Browse Source

[checkpoints][xs]: save data so it's easy to develop new flows.

master
anuveyatsu 4 years ago
parent
commit
8eb6078b2e
2 changed files with 4 additions and 0 deletions
  1. +3
    -0
      .gitignore
  2. +1
    -0
      process.py

+ 3
- 0
.gitignore View File

@ -144,3 +144,6 @@ static
# put you virtual env names here # put you virtual env names here
env36/ env36/
# dataflows checkpoints
.checkpoints/

+ 1
- 0
process.py View File

@ -23,6 +23,7 @@ Flow(
load(f'{BASE_URL}{CONFIRMED}'), load(f'{BASE_URL}{CONFIRMED}'),
load(f'{BASE_URL}{RECOVERED}'), load(f'{BASE_URL}{RECOVERED}'),
load(f'{BASE_URL}{DEATH}'), load(f'{BASE_URL}{DEATH}'),
checkpoint('load_data'),
unpivot(unpivoting_fields, extra_keys, extra_value), unpivot(unpivoting_fields, extra_keys, extra_value),
find_replace([{'name': 'Date', 'patterns': [{'find': '/', 'replace': '-'}]}]), find_replace([{'name': 'Date', 'patterns': [{'find': '/', 'replace': '-'}]}]),
to_normal_date, to_normal_date,


Loading…
Cancel
Save