From 8335e2b1f5b42984718a89fe250d2f3565d12604 Mon Sep 17 00:00:00 2001 From: Adar Nimrod <nimrod@shore.co.il> Date: Wed, 14 Aug 2019 23:55:44 +0300 Subject: [PATCH] A commit that's just too big. Too many changes in one commit that aren't relevant to each other all because I didn't commit these for some time and they piled up, exactly what I shouldn't do. - Remove some Git submodules. They're not needed. - Remove the REPREPRO_BASE_DIR environment variable, it's set with direnv inside that project. - Use Docker Buildkit. - Use auto Python interpeter with Ansible, prefer Python3. - Better ls aliases. - Netcat aliases to listen and check ports and sockets. - Aliases and functions for Terraform. - Use Black to format Python code. - Show the run duration and exit code in a line before the prompt, maybe better with Bash earlier than 5.0, where there's an annoying with line wrapping. - Valid name for the flatpak_kill shell function. - Delete user-dirs files, didn't do anything anyway. - Convert README from reStructuredText to Markdown. - Don't install gsutil with Python2, use snap to install the Google SDK. - More pre-commit hooks, update the exiting ones, format the file for yamllint, Use flake8 from it's repo, the one from the hooks repo is deprecated. - Clean the Shore SSH config. - Add the shared AWS keys to the Endless SSH config. --- .bashrc | 66 +++++++++++-- .config/user-dirs.dirs | 15 --- .githooks/post-merge | 1 - .gitmodules | 7 -- .pre-commit-config.yaml | 92 +++++++++++++++---- .ssh/config.d/20_endless | 2 + .ssh/config.d/20_shore | 16 +--- .../Shore/Ansible/ansible-desktop-playbooks | 1 - Documents/Shore/debian-repository | 1 - Documents/Shore/poquita | 1 - Documents/bin/_vagrant-clean | 27 ++++-- Documents/bin/_vagrant-update | 20 ++-- Documents/bin/urldecode | 2 +- Documents/bin/urlencode | 2 +- README.md | 46 ++++++++++ README.rst | 48 ---------- 16 files changed, 209 insertions(+), 138 deletions(-) delete mode 100644 .config/user-dirs.dirs delete mode 160000 Documents/Shore/Ansible/ansible-desktop-playbooks delete mode 160000 Documents/Shore/debian-repository delete mode 160000 Documents/Shore/poquita create mode 100644 README.md delete mode 100644 README.rst diff --git a/.bashrc b/.bashrc index 1341b3a..1067cea 100644 --- a/.bashrc +++ b/.bashrc @@ -2,12 +2,12 @@ # If not running interactively, don't do anything [ -z "$PS1" ] && return +export PS1='\u@\h:\w\$ ' export LANG=en_US.UTF8 export HISTFILE="$HOME/.history" export HISTCONTROL=ignoreboth:erasedups export HISTSIZE=100000 export HISTFILESIZE=100000 -export REPREPRO_BASE_DIR="$HOME/Documents/Shore/debian-repository" export EDITOR=vim export GOPATH="$HOME/Documents/Golang" export PATH="$GOPATH/bin:/usr/lib/go/bin/:$PATH" @@ -33,6 +33,7 @@ export ANSIBLE_CACHE_PLUGIN_CONNECTION="$HOME/.ansible/facts" export ANSIBLE_CALLBACK_WHITELIST="profile_tasks, timer" export ANSIBLE_SSH_CONTROL_PATH="/tmp/ssh-%%h" export ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED=True +export ANSIBLE_PYTHON_INTERPRETER=auto export LYNX_SAVE_SPACE="$HOME/Downloads" export LYNX_TEMP_SPACE="$HOME/.cache/lynx" export VAGRANT_DEFAULT_PROVIDER="virtualbox" @@ -49,10 +50,11 @@ export LESS_TERMCAP_se=$'\E[0m' export LESS_TERMCAP_us=$'\E[01;32m' export LESS_TERMCAP_ue=$'\E[0m' export HELM_HOME="$HOME/.helm" +export DOCKER_BUILDKIT=1 alias ll='ls -lha' -alias la='ls -A' -alias l='ls -CF' +alias la='ls -AF' +alias l='ls -F' alias gcc='gcc --std=c99 -Wall -Wextra -Werror -pedantic' alias dpkglog="grep -v 'status\\|trigproc\\|configure' /var/log/dpkg.log" alias deborphan='deborphan -a --no-show-section --ignore-suggests' @@ -78,7 +80,7 @@ alias tolower='awk "{print tolower(\$0)}"' # shellcheck disable=SC2142 alias toupper='awk "{print toupper(\$0)}"' alias wifi-portal='curl --silent --fail --write-out "%{redirect_url}" --output /dev/null http://detectportal.firefox.com/success.txt' -alias transmission-remote='ssh -fNo ExitOnForwardFailure=yes xbmc.shore.co.il && transmission-remote' +alias transmission-remote='forward xbmc.shore.co.il 9091:localhost:9091 && transmission-remote' alias kpcli='kpcli --kdb ~/Documents/Database.kdbx' alias gen-mac="hexdump -n5 -e '\"02\" 5/1 \":%02X\" \"\\n\"' /dev/urandom" alias clean-swp="find \$HOME/ -name '*.swp' -delete" @@ -113,6 +115,51 @@ alias prune_docker_remote='find ~/.ssh -maxdepth 1 -type s -name "docker_*" -del alias close='ssh -fnNTS ~/.ssh/%C.sock -O exit' alias jjb='jenkins-jobs' alias diff='diff --unified' +alias check_tcp='nc -vzw1' +alias check_unix='nc -Uvzw1' +alias listen_tcp='nc -vlk 0.0.0.0' +alias listen_udp='nc -uvlk 0.0.0.0' +alias listen_unix='nc -Uvlk' +alias ta='terraform apply tfplan' + +if ! command -v notify-send > /dev/null +then + alias notify-send='bell' +elif [ -n "$GIO_LAUNCHED_DESKTOP_FILE" ] +then + # shellcheck disable=SC2139 + alias notify-send="notify-send --hint \"string:desktop-entry:$(basename "$GIO_LAUNCHED_DESKTOP_FILE")\"" +fi + +tp () { + workspace="$(terraform workspace show)" + if [ "$workspace" = "default" ] + then + terraform plan -out tfplan "$@" + else + terraform plan -out tfplan -var-file "$workspace.tfvars" "$@" + fi +} + +taa () { + workspace="$(terraform workspace show)" + if [ "$workspace" = "default" ] + then + terraform apply -auto-approve "$@" + else + terraform apply -auto-approve -var-file "$workspace.tfvars" "$@" + fi +} + +tr () { + workspace="$(terraform workspace show)" + if [ "$workspace" = "default" ] + then + terraform refresh "$@" + else + terraform refresh -var-file "$workspace.tfvars" "$@" + fi +} genpass () { bytes="${1:-32}" @@ -219,7 +266,7 @@ match_ssl_pair () { return "$exitcode" } -flatpak-kill () { +flatpak_kill () { if [ "$#" -lt 1 ] then echo "You must specify application name." >> /dev/stderr @@ -236,15 +283,16 @@ flatpak-kill () { __prompt () { local exitstatus="$?" - local runduration endtime + local runduration endtime pre_prompt ! [ "$(type history 2> /dev/null)" = 'history is a shell builtin' ] || history -a - PS1='\u@\h:\w\$ ' if [ -n "${starttime:-}" ] then endtime="$(date +%s)" runduration="$(( endtime - starttime))" - [ "$runduration" -lt '10' ] || PS1="\\[\\e[1;96m[Run duration: $runduration]\\e[0m\\] $PS1" - [ "$exitstatus" -eq '0' ] || [ -z "${run_command:-}" ] || PS1="\\[\\e[1;91m[Exit status: $exitstatus]\\e[0m\\] $PS1" + pre_prompt='' + [ "$exitstatus" -eq '0' ] || [ -z "${run_command:-}" ] || pre_prompt="$pre_prompt\\e[1;91m[Exit status: $exitstatus]\\e[0m " + [ "$runduration" -lt '5' ] || pre_prompt="$pre_prompt\\e[1;96m[Run duration: $runduration]\\e[0m " + [ -z "$pre_prompt" ] || echo -e "$pre_prompt" unset run_command fi last_command='__prompt' diff --git a/.config/user-dirs.dirs b/.config/user-dirs.dirs deleted file mode 100644 index ea3a3a4..0000000 --- a/.config/user-dirs.dirs +++ /dev/null @@ -1,15 +0,0 @@ -# This file is written by xdg-user-dirs-update -# If you want to change or add directories, just edit the line you're -# interested in. All local changes will be retained on the next run -# Format is XDG_xxx_DIR="$HOME/yyy", where yyy is a shell-escaped -# homedir-relative path, or XDG_xxx_DIR="/yyy", where /yyy is an -# absolute path. No other format is supported. -# -XDG_DESKTOP_DIR="$HOME/Desktop" -XDG_DOWNLOAD_DIR="$HOME/Downloads" -XDG_TEMPLATES_DIR="$HOME/Templates" -XDG_PUBLICSHARE_DIR="$HOME/Public" -XDG_DOCUMENTS_DIR="$HOME/Documents" -XDG_MUSIC_DIR="$HOME/Music" -XDG_PICTURES_DIR="$HOME/Pictures" -XDG_VIDEOS_DIR="$HOME/Videos" diff --git a/.githooks/post-merge b/.githooks/post-merge index dc8b1c4..827f705 100755 --- a/.githooks/post-merge +++ b/.githooks/post-merge @@ -11,7 +11,6 @@ go get github.com/kaorimatz/mysqldump-loader || true go get github.com/lucagrulla/cw || true go get sigs.k8s.io/kind || true echo Installing Python apps >> /dev/stderr -python2 -m pip install --user gsutil python3 -m pip install --user --requirement requirements.txt echo Downloading binary and vendored files, creating generated files >> /dev/stderr git clean -fX .bash_completion.d/ diff --git a/.gitmodules b/.gitmodules index 8479cf9..0f70bcc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,14 +10,7 @@ path = Documents/Shore/bundle_certs url = https://www.shore.co.il/git/bundle_certs/ branch = master -[submodule "Documents/Shore/debian-repository"] - path = Documents/Shore/debian-repository - url = https://www.shore.co.il/git/debian-repository - branch = master [submodule "Documents/Shore/ansible-desktop-playbooks"] path = Documents/Shore/Ansible/ansible-desktop-playbooks url = https://www.shore.co.il/git/ansible-desktop-playbooks branch = master -[submodule "Documents/Shore/poquita"] - path = Documents/Shore/poquita - url = https://www.shore.co.il/git/poquita diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a770884..33bf83f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,29 +1,85 @@ +# vim:ff=unix ts=2 sw=2 ai expandtab --- repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.2.1 hooks: - - id: check-added-large-files - - id: check-executables-have-shebangs - - id: check-merge-conflict - - id: check-symlinks - - id: detect-private-key - - id: trailing-whitespace - - id: flake8 - exclude: pythonrc.py|rabbitmqadmin - - id: requirements-txt-fixer -- repo: https://www.shore.co.il/git/shell-pre-commit/ + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + - id: trailing-whitespace + - id: requirements-txt-fixer + - repo: https://www.shore.co.il/git/shell-pre-commit/ rev: v0.6.0 hooks: - - id: shell-lint - exclude: &excluded_shellscripts \.bash_completion\.d/(docker-machine\.bash|fabric-completion.bash|docker-compose)|\.travis/travis\.sh - - id: shellcheck + - id: shell-lint + exclude: &excluded_shellscripts |- + \.bash_completion\.d/(docker-machine\.bash|fabric-completion.bash|docker-compose)|\.travis/travis\.sh + - id: shellcheck exclude: *excluded_shellscripts - - id: shell-lint - alias: shell-lint .bashrc + - id: shell-lint + name: shell-lint .bashrc files: \.bashrc types: [text] - - id: shellcheck - alias: Shellcheck .bashrc + - id: shellcheck + name: Shellcheck .bashrc files: \.bashrc types: [text] + - repo: https://github.com/adrienverge/yamllint + rev: v1.17.0 + hooks: + - id: yamllint + - repo: https://github.com/amperser/proselint/ + rev: 0.10.1 + hooks: + - id: proselint + types: [plain-text] + exclude: LICENSE + - repo: https://github.com/ambv/black + rev: 18.9b0 + hooks: + - id: black + exclude: &excluded_pythonscripts pythonrc.py|rabbitmqadmin + args: + - | + --line-length=79 + - repo: https://github.com/PyCQA/prospector + rev: 1.1.6.4 + hooks: + - id: prospector + exclude: *excluded_pythonscripts + args: + - |- + --max-line-length=79 + - |- + --tool=pyroma + - |- + --tool=dodgy + additional_dependencies: + - pyroma + - dodgy + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.7 + hooks: + - id: flake8 + exclude: *excluded_pythonscripts + args: + - |- + --max-line-length=79 + additional_dependencies: + - flake8-bugbear + - repo: https://github.com/pre-commit/mirrors-pylint + rev: v2.3.1 + hooks: + - id: pylint + exclude: *excluded_pythonscripts + args: + - |- + --disable=R0801 + - repo: https://github.com/PyCQA/bandit + rev: 1.6.1 + hooks: + - id: bandit + exclude: *excluded_pythonscripts diff --git a/.ssh/config.d/20_endless b/.ssh/config.d/20_endless index e5552f0..9c03977 100644 --- a/.ssh/config.d/20_endless +++ b/.ssh/config.d/20_endless @@ -33,3 +33,5 @@ Host 172.31.*.* 10.*.*.* *.endlessm.com *.endlessm-sf.com endlessm.com User nimrod IdentityFile ~/.ssh/endless_ecdsa IdentityFile ~/.ssh/endless_rsa +IdentityFile ~/.ssh/endless_us-west-2 +IdentityFile ~/.ssh/endless_us-west-1 diff --git a/.ssh/config.d/20_shore b/.ssh/config.d/20_shore index 8cc33ea..cd1fadd 100644 --- a/.ssh/config.d/20_shore +++ b/.ssh/config.d/20_shore @@ -1,31 +1,17 @@ # shore.co.il infrastructure -Host ns1.shore.co.il www.shore.co.il -DynamicForward 8080 - -Host xbmc.shore.co.il -ForwardX11 yes -LocalForward 9091 localhost:9091 +Host cgit.shore.co.il host01.shore.co.il wdr4300.shore.co.il xbmc.shore.co.il ProxyCommand ssh ns1.shore.co.il -W %h:%p Host ns2.shore.co.il User admin -DynamicForward 9999 - -Host cgit.shore.co.il -ProxyCommand ssh ns1.shore.co.il -W %h:%p Host ns3.shore.co.il User root -DynamicForward 8081 - -Host host01.shore.co.il -ProxyCommand ssh ns1.shore.co.il -W %h:%p Host wdr4300.shore.co.il User root -ProxyCommand ssh ns1.shore.co.il -W %h:%p LocalForward 8082 localhost:80 Host *.shore.co.il 192.168.3.* diff --git a/Documents/Shore/Ansible/ansible-desktop-playbooks b/Documents/Shore/Ansible/ansible-desktop-playbooks deleted file mode 160000 index f2e18e5..0000000 --- a/Documents/Shore/Ansible/ansible-desktop-playbooks +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f2e18e5b4d35a951e5490d5929df769c516e93ce diff --git a/Documents/Shore/debian-repository b/Documents/Shore/debian-repository deleted file mode 160000 index 1639468..0000000 --- a/Documents/Shore/debian-repository +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1639468727302074034cbfe684500a174700f313 diff --git a/Documents/Shore/poquita b/Documents/Shore/poquita deleted file mode 160000 index bebddcf..0000000 --- a/Documents/Shore/poquita +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bebddcfe7603dce3f1739055305ec287d9590f63 diff --git a/Documents/bin/_vagrant-clean b/Documents/bin/_vagrant-clean index dad88af..6251fdf 100755 --- a/Documents/bin/_vagrant-clean +++ b/Documents/bin/_vagrant-clean @@ -6,18 +6,25 @@ from parse import parse boxes = {} for line in vagrant.box("list"): - box = parse('\x1b[0m{name:^} ({provider}, {version})\x1b[0m', line) - if box['name'] in boxes: - if box['provider'] in boxes[box['name']]: - boxes[box['name']][box['provider']].append(box['version']) + box = parse("\x1b[0m{name:^} ({provider}, {version})\x1b[0m", line) + if box["name"] in boxes: + if box["provider"] in boxes[box["name"]]: + boxes[box["name"]][box["provider"]].append(box["version"]) else: - boxes[box['name']][box['provider']] = [box['version']] + boxes[box["name"]][box["provider"]] = [box["version"]] else: - boxes[box['name']] = {box['provider']: [box['version']]} + boxes[box["name"]] = {box["provider"]: [box["version"]]} for box in boxes: for provider in boxes[box]: for version in boxes[box][provider][:-1]: - print('Removing box {0} version {1}.'.format(box, version)) - vagrant.box('remove', '--force', '--provider', provider, - '--box-version', version, box) -vagrant('global-status', '--prune') + print("Removing box {0} version {1}.".format(box, version)) + vagrant.box( + "remove", + "--force", + "--provider", + provider, + "--box-version", + version, + box, + ) +vagrant("global-status", "--prune") diff --git a/Documents/bin/_vagrant-update b/Documents/bin/_vagrant-update index bcaa2c2..141f1c9 100755 --- a/Documents/bin/_vagrant-update +++ b/Documents/bin/_vagrant-update @@ -5,16 +5,16 @@ from parse import parse boxes = {} -dummy_providers = ['aws'] -for line in vagrant.box('list'): - box = parse('\x1b[0m{name:^} ({provider}, {version})\x1b[0m', line) - if not box['provider'] in dummy_providers: - if box['name'] in boxes: - if not box['provider'] in boxes[box['name']]: - boxes[box['name']].append(box['provider']) +dummy_providers = ["aws"] +for line in vagrant.box("list"): + box = parse("\x1b[0m{name:^} ({provider}, {version})\x1b[0m", line) + if not box["provider"] in dummy_providers: + if box["name"] in boxes: + if not box["provider"] in boxes[box["name"]]: + boxes[box["name"]].append(box["provider"]) else: - boxes[box['name']] = [box['provider']] + boxes[box["name"]] = [box["provider"]] for box in boxes: for provider in boxes[box]: - print('Updating box {0} provider {1}.'.format(box, provider)) - vagrant.box('update', '--provider', provider, '--box', box) + print("Updating box {0} provider {1}.".format(box, provider)) + vagrant.box("update", "--provider", provider, "--box", box) diff --git a/Documents/bin/urldecode b/Documents/bin/urldecode index 87954fe..181f14e 100755 --- a/Documents/bin/urldecode +++ b/Documents/bin/urldecode @@ -3,6 +3,6 @@ from fileinput import input from urllib.parse import unquote_plus as unquote -if __name__ == '__main__': +if __name__ == "__main__": for line in input(): print(unquote(line.strip())) diff --git a/Documents/bin/urlencode b/Documents/bin/urlencode index 3f93145..5b1af72 100755 --- a/Documents/bin/urlencode +++ b/Documents/bin/urlencode @@ -3,6 +3,6 @@ from fileinput import input from urllib.parse import quote_plus as quote -if __name__ == '__main__': +if __name__ == "__main__": for line in input(): print(quote(line.strip())) diff --git a/README.md b/README.md new file mode 100644 index 0000000..3803314 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# rcfiles + +[](https://travis-ci.org/adarnimrod/rcfiles) + +A repository with my rc files. The purpose is for me to have revision +control of my home directory, maybe send somebody a link to an example +file and to be browsable for others (if anyone wants to copy some +snippet). Because these are my actual files that I use right now, the +repository is cloned directly to my home directory and is not meant for +mass consumption as it is. + +## Installation + +*WARNING: This will overwrite your existing files.* + +``` +cd +git clone --bare https://www.shore.co.il/git/rcfiles .git +sed -i '/bare/d' .git/config +git reset --hard +git bull +Documents/bin/install-git-hooks +``` + +## Dependencies + +Dependencies that can be installed locally inside the home directory, are +installed with the Git hook using the supplied `Makefile`. Dependencies that +can't be installed locally, can be install with the `workstation.yml` Ansible +playbook from the +[ansible-desktop-playbook](https://www.shore.co.il/git/ansible-desktop-playbooks) +repository, please consult the README from that repository. Care has been given +to minimizing the dependencies and making the scripts as cross-platform as +reasonably possible, so most script would run without any installing any tools +not found on a Unix-like OS by default. + +## License + +This software is licensed under the MIT license (see `LICENSE.txt`). + +## Author Information + +Nimrod Adar, [contact me](mailto:nimrod@shore.co.il) or visit my [website]( +https://www.shore.co.il/). Patches are welcome via [`git send-email`]( +http://git-scm.com/book/en/v2/Git-Commands-Email). The repository is located +at: <https://www.shore.co.il/git/>. diff --git a/README.rst b/README.rst deleted file mode 100644 index 08cf774..0000000 --- a/README.rst +++ /dev/null @@ -1,48 +0,0 @@ -rcfiles -******* - -.. image:: https://travis-ci.org/adarnimrod/rcfiles.svg?branch=master - :target: https://travis-ci.org/adarnimrod/rcfiles - -A repository with my rc files. The purpose is for me to have revision control of -my home directory, maybe send somebody a link to an example file and to be -browsable for others (if anyone wants to copy some snippet). Because these are -my actual files that I use right now, the repository is cloned directly to my -home directory and is not meant for mass consumption as it is. - -Installation ------------- - -*WARNING: This will overwrite your existing files.* - -.. code:: shell - - cd - git clone --bare https://www.shore.co.il/git/rcfiles .git - sed -i '/bare/d' .git/config - git reset --hard - git bull - Documents/bin/install-git-hooks - -Dependencies ------------- - -All of the dependencies are installed with the workstation playbooks inside the -ansible-desktop-playbooks submodule, please consult the README in the submodule. -Care has been given to minimizing the dependencies and making the scripts as -cross-platform as reasonably possible, so most script would run without any -installing any tools not found on a Unix-like OS by default. - -License -------- - -This software is licensed under the MIT license (see the :code:`LICENSE.txt` -file). - -Author Information ------------------- - -Nimrod Adar, `contact me <nimrod@shore.co.il>`_ or visit my `website -<https://www.shore.co.il/>`_. Patches are welcome via `git send-email -<http://git-scm.com/book/en/v2/Git-Commands-Email>`_. The repository is located -at: https://www.shore.co.il/git/. -- GitLab