From 64b2a64f45887d5115c94f39ae710c0976932958 Mon Sep 17 00:00:00 2001
From: Adar Nimrod <nimrod@shore.co.il>
Date: Tue, 18 Jun 2019 15:36:03 +0300
Subject: [PATCH] A big cleanup.
- Use Pipenv for local development tasks in a virtualenv.
- Replace fabfile with Pipfile "scripts".
- Update .gitignore.
- Add lots of checks using pre-commit.
- Format code using Black.
- Use PEP 396 version tracking.
- Use bumpversion for tagging new versions.
- Fix issues found with Flake8, pylint.
---
.gitignore | 25 ++++-
.pre-commit-config.yaml | 52 +++++++++
MANIFEST.in | 1 -
Pipfile | 20 ++++
Pipfile.lock | 215 ++++++++++++++++++++++++++++++++++++
VERSION | 1 -
check_s3_bucket/__init__.py | 124 +++++++++++++--------
fabfile.py | 26 -----
setup.cfg | 11 +-
setup.py | 53 +++++----
10 files changed, 428 insertions(+), 100 deletions(-)
create mode 100644 .pre-commit-config.yaml
create mode 100644 Pipfile
create mode 100644 Pipfile.lock
delete mode 100644 VERSION
delete mode 100644 fabfile.py
diff --git a/.gitignore b/.gitignore
index 2c6ecf8..3e66e1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,22 +11,39 @@ Thumbs.db
.svn/
.sass-cache/
*.log
-a.out
-node-modules/
+*.out
+*.so
+node_modules/
+.npm/
nbproject/
*.ipynb
.idea/
*.egg-info/
-*.o
+*.[ao]
.classpath
.cache/
bower_components/
*.class
-*.jar
+*.[ewj]ar
secring.*
.*.kate-swp
.swp.*
.directory
.Trash-*
build/
+_build/
dist/
+.tox/
+*.pdf
+*.exe
+*.dll
+*.gz
+*.tgz
+*.tar
+*.rar
+*.zip
+*.pid
+*.lock
+*.env
+.bundle/
+!Pipfile.lock
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..f185f9e
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,52 @@
+# vim:ff=unix ts=2 sw=2 ai expandtab
+---
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.2.3
+ hooks:
+ - id: check-added-large-files
+ - id: check-executables-have-shebangs
+ - id: check-merge-conflict
+ - id: detect-private-key
+ - id: trailing-whitespace
+ - repo: https://github.com/ambv/black
+ rev: 18.9b0
+ hooks:
+ - id: black
+ args:
+ - |
+ --line-length=79
+ - repo: https://github.com/Lucas-C/pre-commit-hooks-markup
+ rev: v1.0.0
+ hooks:
+ - id: rst-linter
+ - repo: https://github.com/PyCQA/prospector
+ rev: 1.1.6.4
+ hooks:
+ - id: prospector
+ args:
+ - |-
+ --max-line-length=79
+ - |-
+ --tool=pyroma
+ - |-
+ --tool=dodgy
+ additional_dependencies:
+ - pyroma
+ - dodgy
+ - repo: https://gitlab.com/pycqa/flake8
+ rev: 3.7.7
+ hooks:
+ - id: flake8
+ args:
+ - '--max-line-length=79'
+ additional_dependencies:
+ - flake8-bugbear
+ - repo: https://github.com/pre-commit/mirrors-pylint
+ rev: v2.3.1
+ hooks:
+ - id: pylint
+ - repo: https://github.com/adrienverge/yamllint
+ rev: v1.16.0
+ hooks:
+ - id: yamllint
diff --git a/MANIFEST.in b/MANIFEST.in
index d43dd18..e0361b2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,4 @@
recursive-include check_s3_bucket *.py
exclude .pre-commit-config.yaml
include *.rst
-include VERSION
include *.txt
diff --git a/Pipfile b/Pipfile
new file mode 100644
index 0000000..8e1c9c8
--- /dev/null
+++ b/Pipfile
@@ -0,0 +1,20 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+pre-commit = "*"
+bumpversion = "*"
+twine = "*"
+
+[packages]
+
+[requires]
+python_version = "3.7"
+
+[scripts]
+ci = "pre-commit run --all-files"
+build = "python setup.py bdist_wheel"
+clean = "git clean -fdX"
+upload = "twine upload -s dist/*"
diff --git a/Pipfile.lock b/Pipfile.lock
new file mode 100644
index 0000000..2020b4d
--- /dev/null
+++ b/Pipfile.lock
@@ -0,0 +1,215 @@
+{
+ "_meta": {
+ "hash": {
+ "sha256": "bbe23c390e858bdb748ab771e9f922f21dfbb7ae59874e97a2bf467d9a693272"
+ },
+ "pipfile-spec": 6,
+ "requires": {
+ "python_version": "3.7"
+ },
+ "sources": [
+ {
+ "name": "pypi",
+ "url": "https://pypi.org/simple",
+ "verify_ssl": true
+ }
+ ]
+ },
+ "default": {},
+ "develop": {
+ "aspy.yaml": {
+ "hashes": [
+ "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc",
+ "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45"
+ ],
+ "version": "==1.3.0"
+ },
+ "bleach": {
+ "hashes": [
+ "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16",
+ "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa"
+ ],
+ "version": "==3.1.0"
+ },
+ "bumpversion": {
+ "hashes": [
+ "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e",
+ "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57"
+ ],
+ "index": "pypi",
+ "version": "==0.5.3"
+ },
+ "certifi": {
+ "hashes": [
+ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
+ "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+ ],
+ "version": "==2019.6.16"
+ },
+ "cfgv": {
+ "hashes": [
+ "sha256:32edbe09de6f4521224b87822103a8c16a614d31a894735f7a5b3bcf0eb3c37e",
+ "sha256:3bd31385cd2bebddbba8012200aaf15aa208539f1b33973759b4d02fc2148da5"
+ ],
+ "version": "==2.0.0"
+ },
+ "chardet": {
+ "hashes": [
+ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+ "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+ ],
+ "version": "==3.0.4"
+ },
+ "docutils": {
+ "hashes": [
+ "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
+ "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274",
+ "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"
+ ],
+ "version": "==0.14"
+ },
+ "identify": {
+ "hashes": [
+ "sha256:0a11379b46d06529795442742a043dc2fa14cd8c995ae81d1febbc5f1c014c87",
+ "sha256:43a5d24ffdb07bc7e21faf68b08e9f526a1f41f0056073f480291539ef961dfd"
+ ],
+ "version": "==1.4.5"
+ },
+ "idna": {
+ "hashes": [
+ "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+ "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
+ ],
+ "version": "==2.8"
+ },
+ "importlib-metadata": {
+ "hashes": [
+ "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
+ "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
+ ],
+ "version": "==0.18"
+ },
+ "nodeenv": {
+ "hashes": [
+ "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a"
+ ],
+ "version": "==1.3.3"
+ },
+ "pkginfo": {
+ "hashes": [
+ "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb",
+ "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32"
+ ],
+ "version": "==1.5.0.1"
+ },
+ "pre-commit": {
+ "hashes": [
+ "sha256:92e406d556190503630fd801958379861c94884693a032ba66629d0351fdccd4",
+ "sha256:cccc39051bc2457b0c0f7152a411f8e05e3ba2fe1a5613e4ee0833c1c1985ce3"
+ ],
+ "index": "pypi",
+ "version": "==1.17.0"
+ },
+ "pygments": {
+ "hashes": [
+ "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127",
+ "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"
+ ],
+ "version": "==2.4.2"
+ },
+ "pyyaml": {
+ "hashes": [
+ "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
+ "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
+ "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
+ "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
+ "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
+ "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
+ "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
+ "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
+ "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
+ "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
+ "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
+ ],
+ "version": "==5.1.1"
+ },
+ "readme-renderer": {
+ "hashes": [
+ "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f",
+ "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d"
+ ],
+ "version": "==24.0"
+ },
+ "requests": {
+ "hashes": [
+ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+ "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
+ ],
+ "version": "==2.22.0"
+ },
+ "requests-toolbelt": {
+ "hashes": [
+ "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+ "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+ ],
+ "version": "==0.9.1"
+ },
+ "six": {
+ "hashes": [
+ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+ "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+ ],
+ "version": "==1.12.0"
+ },
+ "toml": {
+ "hashes": [
+ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
+ "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+ ],
+ "version": "==0.10.0"
+ },
+ "tqdm": {
+ "hashes": [
+ "sha256:0a860bf2683fdbb4812fe539a6c22ea3f1777843ea985cb8c3807db448a0f7ab",
+ "sha256:e288416eecd4df19d12407d0c913cbf77aa8009d7fddb18f632aded3bdbdda6b"
+ ],
+ "version": "==4.32.1"
+ },
+ "twine": {
+ "hashes": [
+ "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446",
+ "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc"
+ ],
+ "index": "pypi",
+ "version": "==1.13.0"
+ },
+ "urllib3": {
+ "hashes": [
+ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+ "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+ ],
+ "version": "==1.25.3"
+ },
+ "virtualenv": {
+ "hashes": [
+ "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a",
+ "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783"
+ ],
+ "version": "==16.6.1"
+ },
+ "webencodings": {
+ "hashes": [
+ "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+ "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+ ],
+ "version": "==0.5.1"
+ },
+ "zipp": {
+ "hashes": [
+ "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d",
+ "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3"
+ ],
+ "version": "==0.5.1"
+ }
+ }
+}
diff --git a/VERSION b/VERSION
deleted file mode 100644
index 6e8bf73..0000000
--- a/VERSION
+++ /dev/null
@@ -1 +0,0 @@
-0.1.0
diff --git a/check_s3_bucket/__init__.py b/check_s3_bucket/__init__.py
index da18086..f08a77e 100755
--- a/check_s3_bucket/__init__.py
+++ b/check_s3_bucket/__init__.py
@@ -1,82 +1,120 @@
#!/usr/bin/env python
-from __future__ import (absolute_import, division, print_function,
- unicode_literals)
+"""Check that a filename matching the regex was added to the bucket in the
+given time window."""
+from __future__ import (
+ absolute_import,
+ division,
+ print_function,
+ unicode_literals,
+)
import datetime
import re
import argparse
+
try:
import botocore.session
except ImportError:
- print('Failed to import botocore.')
+ print("Failed to import botocore.")
exit(3)
try:
from dateutil.tz import tzlocal
except ImportError:
- print('Failed to import dateutil.')
+ print("Failed to import dateutil.")
exit(3)
+__version__ = "0.1.0"
+
-def getFileList(bucket):
+def get_file_list(bucket):
+ """Return a list of files in the S3 bucket."""
session = botocore.session.get_session()
- s3client = session.create_client('s3')
+ s3client = session.create_client("s3")
# I'm not concerened with the limitation of number of keys in the
# response as the buckets have a lifecycle rule enabled and files are
# automatically moved of the bucket.
response = s3client.list_objects(Bucket=bucket)
- return response['Contents']
+ return response["Contents"]
def main():
+ """Main entrypoint."""
parser = argparse.ArgumentParser(
- description='''Check that a filename matching the regex was added to the
- bucket in the given time window.''')
- parser.add_argument('bucket', help='S3 bucket to check')
- parser.add_argument('regex',
- help='Filename regex to check (defaults to *)',
- nargs='?',
- default='*')
- parser.add_argument('warning_threshold',
- help='Warning threshold in hours (defaults to 25)',
- default=24,
- type=int,
- nargs='?')
- parser.add_argument('critical_threshold',
- help='Critical threshold in hours (defaults to 49)',
- default=48,
- type=int,
- nargs='?')
+ description="""Check that a filename matching the regex was added to the
+ bucket in the given time window."""
+ )
+ parser.add_argument("bucket", help="S3 bucket to check")
+ parser.add_argument(
+ "regex",
+ help="Filename regex to check (defaults to *)",
+ nargs="?",
+ default="*",
+ )
+ parser.add_argument(
+ "warning_threshold",
+ help="Warning threshold in hours (defaults to 25)",
+ default=24,
+ type=int,
+ nargs="?",
+ )
+ parser.add_argument(
+ "critical_threshold",
+ help="Critical threshold in hours (defaults to 49)",
+ default=48,
+ type=int,
+ nargs="?",
+ )
args = parser.parse_args()
try:
- filelist = getFileList(args.bucket)
- except BaseException as e:
- print('Failed to list file in bucket.')
+ filelist = get_file_list(args.bucket)
+ # pylint: disable=broad-except
+ except BaseException as exception:
+ assert exception
+ print("Failed to list files in bucket.")
exit(3)
- if args.regex != '*':
- p = re.compile(args.regex)
- filelist = filter(lambda x: p.search(x['Key']) is not None, filelist)
- if len(filelist) == 0:
- print('No files matching "{}" found in {}.'.format(args.regex,
- args.bucket))
+ if args.regex != "*":
+ regex = re.compile(args.regex)
+ filelist = filter(
+ lambda x: regex.search(x["Key"]) is not None, filelist
+ )
+ if not filelist:
+ print(
+ 'No files matching "{}" found in {}.'.format(
+ args.regex, args.bucket
+ )
+ )
exit(1)
now = datetime.datetime.now(tz=tzlocal())
- LastModifiedDeltas = map(
- lambda x: int((now - x['LastModified']).total_seconds() / 3600),
- filelist)
+ # pylint: disable=invalid-name
+ LastModifiedDeltas = list(
+ map(
+ lambda x: int((now - x["LastModified"]).total_seconds() / 3600),
+ filelist,
+ )
+ )
LastModifiedDeltas.sort()
delta = LastModifiedDeltas[0]
if delta >= args.critical_threshold:
- print('Last file modified is older than {} hours.'.format(
- args.critical_threshold))
+ print(
+ "Last file modified is older than {} hours.".format(
+ args.critical_threshold
+ )
+ )
exit(2)
elif delta >= args.warning_threshold:
- print('Last file modified is older than {} hours.'.format(
- args.warning_threshold))
+ print(
+ "Last file modified is older than {} hours.".format(
+ args.warning_threshold
+ )
+ )
exit(1)
else:
- print('Last file modified is newer than {} hours.'.format(
- args.warning_threshold))
+ print(
+ "Last file modified is newer than {} hours.".format(
+ args.warning_threshold
+ )
+ )
exit(0)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/fabfile.py b/fabfile.py
deleted file mode 100644
index 85079b4..0000000
--- a/fabfile.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from __future__ import (absolute_import, division, print_function,
- unicode_literals)
-from fabric.api import (local, task, sudo, env, settings)
-
-env.use_ssh_config = True
-
-
-@task
-def build():
- '''Build wheel.'''
- local('''python setup.py sdist bdist_wheel''')
-
-
-@task
-def clean():
- '''Clean cache, build files.'''
- local('''rm -rf *.egg-info build dist''')
- local('''find -name *.pyc -delete''')
- local('''find -name __pycache__ -type d -delete''')
-
-
-@task
-def upload():
- '''Build and upload to PyPI.'''
- build()
- local('''twine upload -s dist/*''')
diff --git a/setup.cfg b/setup.cfg
index a26fcc2..dc2c01f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,15 @@
[bdist_wheel]
-universal=1
+universal = 1
[flake8]
exclude = .tox,*.egg,build,data
select = E,W,F
+
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+
+[bumpversion:file:setup.py]
+
+[bumpversion:file:check_s3_bucket/__init__.py]
diff --git a/setup.py b/setup.py
index 81ef96d..9a63736 100644
--- a/setup.py
+++ b/setup.py
@@ -1,27 +1,32 @@
#!/usr/bin/env python
+# pylint: disable=missing-docstring
from setuptools import setup, find_packages
-setup(name='check_s3_bucket',
- version=open('VERSION', 'r').read(),
- description='''Check that a filename matching the regex was added to the
- bucket in the given time window.''',
- long_description=open('README.rst', 'r').read(),
- url='https://www.shore.co.il/git/check_s3_bucket',
- author='Nimrod Adar',
- author_email='nimrod@shore.co.il',
- license='MIT',
- classifiers=[
- 'Development Status :: 4 - Beta',
- 'Intended Audience :: System Administrators',
- 'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 2', 'Topic :: Utilities',
- 'License :: OSI Approved :: MIT License'
- ],
- keywords='nagios s3 aws monitoring',
- packages=find_packages(),
- install_requires=['python-dateutil', 'botocore'],
- entry_points={
- 'console_scripts': [
- 'check_s3_bucket=check_s3_bucket:main'
- ],
- }, )
+setup(
+ name="check_s3_bucket",
+ version="0.1.0",
+ description="""Check that a filename matching the regex was added to the
+ bucket in the given time window.""",
+ long_description=open("README.rst", "r").read(),
+ url="https://www.shore.co.il/git/check_s3_bucket",
+ author="Nimrod Adar",
+ author_email="nimrod@shore.co.il",
+ license="MIT",
+ classifiers=[
+ "Development Status :: 4 - Beta",
+ "Intended Audience :: System Administrators",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.4",
+ "Programming Language :: Python :: 3.5",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 2",
+ "Programming Language :: Python :: 2.7",
+ "Topic :: Utilities",
+ "License :: OSI Approved :: MIT License",
+ ],
+ keywords="nagios s3 aws monitoring",
+ packages=find_packages(),
+ install_requires=["python-dateutil", "botocore"],
+ entry_points={"console_scripts": ["check_s3_bucket=check_s3_bucket:main"]},
+)
--
GitLab