A big cleanup.

- Use Pipenv for local development tasks in a virtualenv. - Replace fabfile with Pipfile "scripts". - Update .gitignore. - Add lots of checks using pre-commit. - Format code using Black. - Use PEP 396 version tracking. - Use bumpversion for tagging new versions. - Fix issues found with Flake8, pylint.

A big cleanup.
64b2a64f · nimrod · c7f2235a · 64b2a64f · 64b2a64f · 64b2a64f
Commit 64b2a64f authored 5 years ago by nimrod
--- a/.gitignore
+++ b/.gitignore
@@ -11,22 +11,39 @@ Thumbs.db
 .svn/
 .sass-cache/
 *.log
-a.out
-node-modules/
+*.out
+*.so
+node_modules/
+.npm/
 nbproject/
 *.ipynb
 .idea/
 *.egg-info/
-*.o
+*.[ao]
 .classpath
 .cache/
 bower_components/
 *.class
-*.jar
+*.[ewj]ar
 secring.*
 .*.kate-swp
 .swp.*
 .directory
 .Trash-*
 build/
+_build/
 dist/
+.tox/
+*.pdf
+*.exe
+*.dll
+*.gz
+*.tgz
+*.tar
+*.rar
+*.zip
+*.pid
+*.lock
+*.env
+.bundle/
+!Pipfile.lock
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
+# vim:ff=unix ts=2 sw=2 ai expandtab
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.2.3
+    hooks:
+      - id: check-added-large-files
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: trailing-whitespace
+  - repo: https://github.com/ambv/black
+    rev: 18.9b0
+    hooks:
+      - id: black
+        args:
+          - |
+              --line-length=79
+  - repo: https://github.com/Lucas-C/pre-commit-hooks-markup
+    rev: v1.0.0
+    hooks:
+      - id: rst-linter
+  - repo: https://github.com/PyCQA/prospector
+    rev: 1.1.6.4
+    hooks:
+      - id: prospector
+        args:
+          - |-
+              --max-line-length=79
+          - |-
+              --tool=pyroma
+          - |-
+              --tool=dodgy
+        additional_dependencies:
+          - pyroma
+          - dodgy
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.7
+    hooks:
+      - id: flake8
+        args:
+          - '--max-line-length=79'
+        additional_dependencies:
+          - flake8-bugbear
+  - repo: https://github.com/pre-commit/mirrors-pylint
+    rev: v2.3.1
+    hooks:
+      - id: pylint
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.16.0
+    hooks:
+      - id: yamllint
--- a/MANIFEST.in
+++ b/MANIFEST.in
 recursive-include check_s3_bucket *.py
 exclude .pre-commit-config.yaml
 include *.rst
-include VERSION
 include *.txt
--- a/Pipfile
+++ b/Pipfile
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pre-commit = "*"
+bumpversion = "*"
+twine = "*"
+
+[packages]
+
+[requires]
+python_version = "3.7"
+
+[scripts]
+ci = "pre-commit run --all-files"
+build = "python setup.py bdist_wheel"
+clean = "git clean -fdX"
+upload = "twine upload -s dist/*"
--- a/Pipfile.lock
+++ b/Pipfile.lock
+{
+    "_meta": {
+        "hash": {
+            "sha256": "bbe23c390e858bdb748ab771e9f922f21dfbb7ae59874e97a2bf467d9a693272"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {},
+    "develop": {
+        "aspy.yaml": {
+            "hashes": [
+                "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc",
+                "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45"
+            ],
+            "version": "==1.3.0"
+        },
+        "bleach": {
+            "hashes": [
+                "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16",
+                "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa"
+            ],
+            "version": "==3.1.0"
+        },
+        "bumpversion": {
+            "hashes": [
+                "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e",
+                "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57"
+            ],
+            "index": "pypi",
+            "version": "==0.5.3"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
+                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+            ],
+            "version": "==2019.6.16"
+        },
+        "cfgv": {
+            "hashes": [
+                "sha256:32edbe09de6f4521224b87822103a8c16a614d31a894735f7a5b3bcf0eb3c37e",
+                "sha256:3bd31385cd2bebddbba8012200aaf15aa208539f1b33973759b4d02fc2148da5"
+            ],
+            "version": "==2.0.0"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "docutils": {
+            "hashes": [
+                "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
+                "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274",
+                "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"
+            ],
+            "version": "==0.14"
+        },
+        "identify": {
+            "hashes": [
+                "sha256:0a11379b46d06529795442742a043dc2fa14cd8c995ae81d1febbc5f1c014c87",
+                "sha256:43a5d24ffdb07bc7e21faf68b08e9f526a1f41f0056073f480291539ef961dfd"
+            ],
+            "version": "==1.4.5"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+            ],
+            "version": "==2.8"
+        },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
+                "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
+            ],
+            "version": "==0.18"
+        },
+        "nodeenv": {
+            "hashes": [
+                "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a"
+            ],
+            "version": "==1.3.3"
+        },
+        "pkginfo": {
+            "hashes": [
+                "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb",
+                "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32"
+            ],
+            "version": "==1.5.0.1"
+        },
+        "pre-commit": {
+            "hashes": [
+                "sha256:92e406d556190503630fd801958379861c94884693a032ba66629d0351fdccd4",
+                "sha256:cccc39051bc2457b0c0f7152a411f8e05e3ba2fe1a5613e4ee0833c1c1985ce3"
+            ],
+            "index": "pypi",
+            "version": "==1.17.0"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127",
+                "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"
+            ],
+            "version": "==2.4.2"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
+                "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
+                "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
+                "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
+                "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
+                "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
+                "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
+                "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
+                "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
+                "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
+                "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
+            ],
+            "version": "==5.1.1"
+        },
+        "readme-renderer": {
+            "hashes": [
+                "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f",
+                "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d"
+            ],
+            "version": "==24.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+            ],
+            "version": "==2.22.0"
+        },
+        "requests-toolbelt": {
+            "hashes": [
+                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+            ],
+            "version": "==0.9.1"
+        },
+        "six": {
+            "hashes": [
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+            ],
+            "version": "==1.12.0"
+        },
+        "toml": {
+            "hashes": [
+                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
+                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+            ],
+            "version": "==0.10.0"
+        },
+        "tqdm": {
+            "hashes": [
+                "sha256:0a860bf2683fdbb4812fe539a6c22ea3f1777843ea985cb8c3807db448a0f7ab",
+                "sha256:e288416eecd4df19d12407d0c913cbf77aa8009d7fddb18f632aded3bdbdda6b"
+            ],
+            "version": "==4.32.1"
+        },
+        "twine": {
+            "hashes": [
+                "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446",
+                "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc"
+            ],
+            "index": "pypi",
+            "version": "==1.13.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+            ],
+            "version": "==1.25.3"
+        },
+        "virtualenv": {
+            "hashes": [
+                "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a",
+                "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783"
+            ],
+            "version": "==16.6.1"
+        },
+        "webencodings": {
+            "hashes": [
+                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+            ],
+            "version": "==0.5.1"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d",
+                "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3"
+            ],
+            "version": "==0.5.1"
+        }
+    }
+}
--- a/VERSION
+++ b/VERSION
-0.1.0
--- a/check_s3_bucket/__init__.py
+++ b/check_s3_bucket/__init__.py
 #!/usr/bin/env python
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
+"""Check that a filename matching the regex was added to the bucket in the
+given time window."""
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
 import datetime
 import re
 import argparse
+
 try:
    import botocore.session
 except ImportError:
-    print('Failed to import botocore.')
+    print("Failed to import botocore.")
    exit(3)
 try:
    from dateutil.tz import tzlocal
 except ImportError:
-    print('Failed to import dateutil.')
+    print("Failed to import dateutil.")
    exit(3)

+__version__ = "0.1.0"
+

-def getFileList(bucket):
+def get_file_list(bucket):
+    """Return a list of files in the S3 bucket."""
    session = botocore.session.get_session()
-    s3client = session.create_client('s3')
+    s3client = session.create_client("s3")
    # I'm not concerened with the limitation of number of keys in the
    # response as the buckets have a lifecycle rule enabled and files are
    # automatically moved of the bucket.
    response = s3client.list_objects(Bucket=bucket)
-    return response['Contents']
+    return response["Contents"]


 def main():
+    """Main entrypoint."""
    parser = argparse.ArgumentParser(
-        description='''Check that a filename matching the regex was added to the
-        bucket in the given time window.''')
-    parser.add_argument('bucket', help='S3 bucket to check')
-    parser.add_argument('regex',
-                        help='Filename regex to check (defaults to *)',
-                        nargs='?',
-                        default='*')
-    parser.add_argument('warning_threshold',
-                        help='Warning threshold in hours (defaults to 25)',
+        description="""Check that a filename matching the regex was added to the
+        bucket in the given time window."""
+    )
+    parser.add_argument("bucket", help="S3 bucket to check")
+    parser.add_argument(
+        "regex",
+        help="Filename regex to check (defaults to *)",
+        nargs="?",
+        default="*",
+    )
+    parser.add_argument(
+        "warning_threshold",
+        help="Warning threshold in hours (defaults to 25)",
        default=24,
        type=int,
-                        nargs='?')
-    parser.add_argument('critical_threshold',
-                        help='Critical threshold in hours (defaults to 49)',
+        nargs="?",
+    )
+    parser.add_argument(
+        "critical_threshold",
+        help="Critical threshold in hours (defaults to 49)",
        default=48,
        type=int,
-                        nargs='?')
+        nargs="?",
+    )
    args = parser.parse_args()
    try:
-        filelist = getFileList(args.bucket)
-    except BaseException as e:
-        print('Failed to list file in bucket.')
+        filelist = get_file_list(args.bucket)
+    # pylint: disable=broad-except
+    except BaseException as exception:
+        assert exception
+        print("Failed to list files in bucket.")
        exit(3)
-    if args.regex != '*':
-        p = re.compile(args.regex)
-        filelist = filter(lambda x: p.search(x['Key']) is not None, filelist)
-    if len(filelist) == 0:
-        print('No files matching "{}" found in {}.'.format(args.regex,
-                                                           args.bucket))
+    if args.regex != "*":
+        regex = re.compile(args.regex)
+        filelist = filter(
+            lambda x: regex.search(x["Key"]) is not None, filelist
+        )
+    if not filelist:
+        print(
+            'No files matching "{}" found in {}.'.format(
+                args.regex, args.bucket
+            )
+        )
        exit(1)
    now = datetime.datetime.now(tz=tzlocal())
-    LastModifiedDeltas = map(
-        lambda x: int((now - x['LastModified']).total_seconds() / 3600),
-        filelist)
+    # pylint: disable=invalid-name
+    LastModifiedDeltas = list(
+        map(
+            lambda x: int((now - x["LastModified"]).total_seconds() / 3600),
+            filelist,
+        )
+    )
    LastModifiedDeltas.sort()
    delta = LastModifiedDeltas[0]
    if delta >= args.critical_threshold:
-        print('Last file modified is older than {} hours.'.format(
-            args.critical_threshold))
+        print(
+            "Last file modified is older than {} hours.".format(
+                args.critical_threshold
+            )
+        )
        exit(2)
    elif delta >= args.warning_threshold:
-        print('Last file modified is older than {} hours.'.format(
-            args.warning_threshold))
+        print(
+            "Last file modified is older than {} hours.".format(
+                args.warning_threshold
+            )
+        )
        exit(1)
    else:
-        print('Last file modified is newer than {} hours.'.format(
-            args.warning_threshold))
+        print(
+            "Last file modified is newer than {} hours.".format(
+                args.warning_threshold
+            )
+        )
        exit(0)


-if __name__ == '__main__':
+if __name__ == "__main__":
    main()
--- a/fabfile.py
+++ b/fabfile.py
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
-from fabric.api import (local, task, sudo, env, settings)
-
-env.use_ssh_config = True
-
-
-@task
-def build():
-    '''Build wheel.'''
-    local('''python setup.py sdist bdist_wheel''')
-
-
-@task
-def clean():
-    '''Clean cache, build files.'''
-    local('''rm -rf *.egg-info build dist''')
-    local('''find -name *.pyc -delete''')
-    local('''find -name __pycache__ -type d -delete''')
-
-
-@task
-def upload():
-    '''Build and upload to PyPI.'''
-    build()
-    local('''twine upload -s dist/*''')
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,3 +4,12 @@ universal=1
 [flake8]
 exclude = .tox,*.egg,build,data
 select = E,W,F
+
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+
+[bumpversion:file:setup.py]
+
+[bumpversion:file:check_s3_bucket/__init__.py]
--- a/setup.py
+++ b/setup.py
 #!/usr/bin/env python
+# pylint: disable=missing-docstring
 from setuptools import setup, find_packages

-setup(name='check_s3_bucket',
-      version=open('VERSION', 'r').read(),
-      description='''Check that a filename matching the regex was added to the
-    bucket in the given time window.''',
-      long_description=open('README.rst', 'r').read(),
-      url='https://www.shore.co.il/git/check_s3_bucket',
-      author='Nimrod Adar',
-      author_email='nimrod@shore.co.il',
-      license='MIT',
+setup(
+    name="check_s3_bucket",
+    version="0.1.0",
+    description="""Check that a filename matching the regex was added to the
+    bucket in the given time window.""",
+    long_description=open("README.rst", "r").read(),
+    url="https://www.shore.co.il/git/check_s3_bucket",
+    author="Nimrod Adar",
+    author_email="nimrod@shore.co.il",
+    license="MIT",
    classifiers=[
-          'Development Status :: 4 - Beta',
-          'Intended Audience :: System Administrators',
-          'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 2', 'Topic :: Utilities',
-          'License :: OSI Approved :: MIT License'
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: System Administrators",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Topic :: Utilities",
+        "License :: OSI Approved :: MIT License",
    ],
-      keywords='nagios s3 aws monitoring',
+    keywords="nagios s3 aws monitoring",
    packages=find_packages(),
-      install_requires=['python-dateutil', 'botocore'],
-      entry_points={
-          'console_scripts': [
-              'check_s3_bucket=check_s3_bucket:main'
-          ],
-      }, )
+    install_requires=["python-dateutil", "botocore"],
+    entry_points={"console_scripts": ["check_s3_bucket=check_s3_bucket:main"]},
+)