Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
C
check_s3_bucket
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
nimrod
check_s3_bucket
Commits
78cd06a8
Commit
78cd06a8
authored
4 years ago
by
nimrod
Browse files
Options
Downloads
Patches
Plain Diff
Address pre-commit issues.
parent
300f1b30
Branches
Branches containing commit
No related tags found
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
.pre-commit-config.yaml
+1
-0
1 addition, 0 deletions
.pre-commit-config.yaml
MANIFEST.in
+1
-0
1 addition, 0 deletions
MANIFEST.in
check_s3_bucket/__init__.py
+16
-15
16 additions, 15 deletions
check_s3_bucket/__init__.py
with
18 additions
and
15 deletions
.pre-commit-config.yaml
+
1
−
0
View file @
78cd06a8
...
@@ -13,6 +13,7 @@ repos:
...
@@ -13,6 +13,7 @@ repos:
rev
:
v0.14.3
rev
:
v0.14.3
hooks
:
hooks
:
-
id
:
detect-secrets
-
id
:
detect-secrets
exclude
:
Pipfile\.lock
-
repo
:
https://github.com/adrienverge/yamllint
-
repo
:
https://github.com/adrienverge/yamllint
rev
:
v1.25.0
rev
:
v1.25.0
...
...
This diff is collapsed.
Click to expand it.
MANIFEST.in
+
1
−
0
View file @
78cd06a8
recursive-include check_s3_bucket *.py
recursive-include check_s3_bucket *.py
exclude .pre-commit-config.yaml
exclude .pre-commit-config.yaml
exclude .gitlab-ci.yml
include *.rst
include *.rst
include *.txt
include *.txt
exclude .dockerignore
exclude .dockerignore
...
...
This diff is collapsed.
Click to expand it.
check_s3_bucket/__init__.py
+
16
−
15
View file @
78cd06a8
...
@@ -9,18 +9,19 @@ from __future__ import (
...
@@ -9,18 +9,19 @@ from __future__ import (
)
)
import
argparse
import
argparse
import
datetime
import
datetime
import
sys
try
:
try
:
import
botocore.session
import
botocore.session
import
botocore.exceptions
import
botocore.exceptions
except
ImportError
:
except
ImportError
:
print
(
"
Failed to import botocore.
"
)
print
(
"
Failed to import botocore.
"
)
exit
(
3
)
sys
.
exit
(
3
)
try
:
try
:
import
pytz
import
pytz
except
ImportError
:
except
ImportError
:
print
(
"
Failed to import pytz.
"
)
print
(
"
Failed to import pytz.
"
)
exit
(
3
)
sys
.
exit
(
3
)
__version__
=
"
0.2.4
"
__version__
=
"
0.2.4
"
NOW
=
datetime
.
datetime
.
now
(
pytz
.
utc
)
NOW
=
datetime
.
datetime
.
now
(
pytz
.
utc
)
...
@@ -44,7 +45,7 @@ def get_file_list(conn, bucket, prefix=""):
...
@@ -44,7 +45,7 @@ def get_file_list(conn, bucket, prefix=""):
return
files
return
files
def
main
():
def
main
():
# noqa: C901
"""
Main entrypoint.
"""
"""
Main entrypoint.
"""
# Parse command line arguments.
# Parse command line arguments.
...
@@ -79,8 +80,8 @@ def main():
...
@@ -79,8 +80,8 @@ def main():
)
)
parser
.
add_argument
(
parser
.
add_argument
(
"
size_critical_threshold
"
,
"
size_critical_threshold
"
,
help
=
"""
Critical threshold for the difference in size between the
latest
help
=
"""
Critical threshold for the difference in size between the
2 files in percents (default to 50)
"""
,
latest
2 files in percents (default to 50)
"""
,
default
=
50
,
default
=
50
,
type
=
int
,
type
=
int
,
nargs
=
"
?
"
,
nargs
=
"
?
"
,
...
@@ -96,16 +97,16 @@ def main():
...
@@ -96,16 +97,16 @@ def main():
except
botocore
.
exceptions
.
BotoCoreError
as
exception
:
except
botocore
.
exceptions
.
BotoCoreError
as
exception
:
print
(
"
Failed to list the files in the S3 bucket.
"
)
print
(
"
Failed to list the files in the S3 bucket.
"
)
print
(
str
(
exception
))
print
(
str
(
exception
))
exit
(
3
)
sys
.
exit
(
3
)
if
not
files
:
if
not
files
:
print
(
"
No matching files in bucket.
"
)
print
(
"
No matching files in bucket.
"
)
exit
(
2
)
sys
.
exit
(
2
)
# Calculate the age of the latest file and if it's in the thresholds set.
# Calculate the age of the latest file and if it's in the thresholds set.
if
files
[
0
][
"
LastModified
"
]
>
NOW
:
if
files
[
0
][
"
LastModified
"
]
>
NOW
:
print
(
"
Latest file is from the future, something is wrong.
"
)
print
(
"
Latest file is from the future, something is wrong.
"
)
exit
(
3
)
sys
.
exit
(
3
)
timedelta
=
files
[
0
][
"
HoursSinceLastModified
"
]
timedelta
=
files
[
0
][
"
HoursSinceLastModified
"
]
if
timedelta
>
args
.
age_critical_threshold
:
if
timedelta
>
args
.
age_critical_threshold
:
print
(
print
(
...
@@ -113,29 +114,29 @@ def main():
...
@@ -113,29 +114,29 @@ def main():
args
.
age_critical_threshold
args
.
age_critical_threshold
)
)
)
)
exit
(
2
)
sys
.
exit
(
2
)
elif
timedelta
>
args
.
age_warning_threshold
:
elif
timedelta
>
args
.
age_warning_threshold
:
print
(
print
(
"
Last file modified is older than {} hours.
"
.
format
(
"
Last file modified is older than {} hours.
"
.
format
(
args
.
age_warning_threshold
args
.
age_warning_threshold
)
)
)
)
exit
(
1
)
sys
.
exit
(
1
)
# Calculate the size ratio between the latest 2 files and check if
# Calculate the size ratio between the latest 2 files and check if
# it's in the threshold set.
# it's in the threshold set.
if
files
[
0
][
"
Size
"
]
==
0
:
if
files
[
0
][
"
Size
"
]
==
0
:
print
(
"
Latest file is empty.
"
)
print
(
"
Latest file is empty.
"
)
exit
(
2
)
sys
.
exit
(
2
)
elif
len
(
files
)
==
1
:
elif
len
(
files
)
==
1
:
print
(
print
(
"""
Found only 1 file in the bucket, can
'
t calculate size
"""
Found only 1 file in the bucket, can
'
t calculate size
difference.
"""
difference.
"""
)
)
exit
(
3
)
sys
.
exit
(
3
)
elif
files
[
1
][
"
Size
"
]
==
0
:
elif
files
[
1
][
"
Size
"
]
==
0
:
print
(
"
The last but 1 file is empty, can
'
t calculate size difference.
"
)
print
(
"
The last but 1 file is empty, can
'
t calculate size difference.
"
)
exit
(
3
)
sys
.
exit
(
3
)
size_ratio
=
100
*
abs
(
size_ratio
=
100
*
abs
(
(
files
[
1
][
"
Size
"
]
-
files
[
0
][
"
Size
"
])
/
files
[
1
][
"
Size
"
]
(
files
[
1
][
"
Size
"
]
-
files
[
0
][
"
Size
"
])
/
files
[
1
][
"
Size
"
]
...
@@ -146,14 +147,14 @@ def main():
...
@@ -146,14 +147,14 @@ def main():
size_ratio
size_ratio
)
)
)
)
exit
(
2
)
sys
.
exit
(
2
)
if
size_ratio
>
args
.
size_warning_threshold
:
if
size_ratio
>
args
.
size_warning_threshold
:
print
(
print
(
"
The size difference between the latest 2 file is {}%.
"
.
format
(
"
The size difference between the latest 2 file is {}%.
"
.
format
(
size_ratio
size_ratio
)
)
)
)
exit
(
1
)
sys
.
exit
(
1
)
else
:
else
:
print
(
"
File found and is within the thresholds set.
"
)
print
(
"
File found and is within the thresholds set.
"
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment