From 388bf67425ca45f071a52f1ae3e6fb8ad4aa695b Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Sun, 22 Mar 2020 20:51:00 +0600 Subject: [PATCH 1/6] [setup][xs]: add requirements.txt --- requirements.txt | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b290592 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,52 @@ +attrs==19.3.0 +awesome-slugify==1.6.5 +bitarray==1.2.1 +bitstring==3.1.6 +blessings==1.7 +boto3==1.12.26 +botocore==1.15.26 +cachetools==4.0.0 +cchardet==2.1.6 +certifi==2019.11.28 +chardet==3.0.4 +click==7.1.1 +dataflows==0.0.71 +datapackage==1.11.1 +docutils==0.15.2 +et-xmlfile==1.0.1 +idna==2.9 +ijson==2.6.1 +importlib-metadata==1.5.0 +inquirer==2.6.3 +isodate==0.6.0 +jdcal==1.4.1 +Jinja2==2.11.1 +jmespath==0.9.5 +jsonlines==1.2.0 +jsonpointer==2.0 +jsonschema==3.2.0 +kvfile==0.0.7 +linear-tsv==1.1.0 +MarkupSafe==1.1.1 +openpyxl==3.0.3 +pybloom-live==3.0.0 +pyrsistent==0.15.7 +python-dateutil==2.8.1 +python-editor==1.0.4 +readchar==2.0.1 +regex==2020.2.20 +requests==2.23.0 +rfc3986==1.3.2 +s3transfer==0.3.3 +six==1.14.0 +SQLAlchemy==1.3.15 +tableschema==1.15.0 +tableschema-sql==1.3.1 +tabulate==0.8.6 +tabulator==1.36.0 +unicodecsv==0.14.1 +Unidecode==0.4.21 +urllib3==1.25.8 +xlrd==1.2.0 +xmljson==0.2.0 +zipp==3.1.0 From bde12232a8cc2b8b38421cbaf61059bab5a2bb6e Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Sun, 22 Mar 2020 20:51:39 +0600 Subject: [PATCH 2/6] [git][xs]: add .gitignore for python projects. --- .gitignore | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d34fa8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,146 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# static files generated from Django application using `collectstatic` +media +static + +# put you virtual env names here +env36/ From 8ca3ead5bec779a003d643ff6f73e3872e8e9541 Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Sun, 22 Mar 2020 20:58:36 +0600 Subject: [PATCH 3/6] [gh actions][s]: update the data automatically and push to github. --- .github/workflows/actions.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/actions.yml diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml new file mode 100644 index 0000000..b191e82 --- /dev/null +++ b/.github/workflows/actions.yml @@ -0,0 +1,27 @@ +on: + push: + branches: + - feature/auto-update-data + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Build the data and create local changes + uses: actions/setup-python@v1 + with: + python-version: '3.6.2' + architecture: x64 + - run: | + pip install -r requirements.txt + python process.py + - name: Commit files + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git commit -m "Daily auto-update of the data" -a + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} From 96eae7c2342d0f43e7e0378a5b0a976f3e430a36 Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Sun, 22 Mar 2020 21:00:51 +0600 Subject: [PATCH 4/6] Use python version 3.6.10 since the previous one isn't available in gh actions. --- .github/workflows/actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index b191e82..9bc90f0 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -11,7 +11,7 @@ jobs: - name: Build the data and create local changes uses: actions/setup-python@v1 with: - python-version: '3.6.2' + python-version: '3.6.10' architecture: x64 - run: | pip install -r requirements.txt From 6904d1d80557c820d40711f9fe51ea299c61b368 Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Sun, 22 Mar 2020 21:25:59 +0600 Subject: [PATCH 5/6] [gh actions][xs]: change secret name. It is required as github doesn't allow underscore in the secret name. --- .github/workflows/actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 9bc90f0..a7bdb0a 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -24,4 +24,4 @@ jobs: - name: Push changes uses: ad-m/github-push-action@master with: - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ secrets.gh }} From 292d6a40bbd29acfa292fe25dda959741013437f Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Sun, 22 Mar 2020 21:45:19 +0600 Subject: [PATCH 6/6] [cron][xs]: run the action on a schedule every midnight GMT. --- .github/workflows/actions.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index a7bdb0a..f75e5d7 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -1,7 +1,6 @@ on: - push: - branches: - - feature/auto-update-data + schedule: + - cron: '0 0 * * *' jobs: deploy: