Skip to content

Set up production necessities for deploying Safegraph #259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 3, 2020
Merged

Set up production necessities for deploying Safegraph #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a3b0677
Add updated encrypted credentials
korlaxxalrok Aug 31, 2020
4b52e78
Add wip_signal to production params template
korlaxxalrok Sep 1, 2020
df73c7b
Add newline
korlaxxalrok Sep 2, 2020
bd59cfa
Run on 12 cores
korlaxxalrok Sep 3, 2020
1ee2208
Add shell script to run the indicator
korlaxxalrok Sep 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions ansible/templates/safegraph-params-prod.json.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
"raw_data_dir": "/common/safegraph",
"export_dir": "./receiving",
"cache_dir": "./cache",
"n_core": "6",
"n_core": "12",
"aws_access_key_id": "{{ safegraph_aws_access_key_id }}",
"aws_secret_access_key": "{{ safegraph_aws_secret_access_key }}",
"aws_default_region": "us-east-1",
"aws_endpoint": "https://s3.wasabisys.com"
}
"aws_endpoint": "https://s3.wasabisys.com",
"wip_signal": ""
}
54 changes: 21 additions & 33 deletions ansible/vault.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,34 +1,22 @@
$ANSIBLE_VAULT;1.1;AES256
<<<<<<< HEAD
30613131613362333037373437316261643833316430643638623461613137346536616262313232
3666623130633831343831353265633338336363376664340a646639323034653632333866616561
35656432653066633064386563356361313737333563376231623134623462663033383636346537
6139373862666635370a393230316563366230326130323135366335376634373364623831323638
34316133303664373135633331383530353362323263313932346232666164333033666564313061
33346637363561633234356137633364633661356433643839363363383662396437363761653261
35366164376466373631363532313337666230656162316561343263336335633164353565333565
36613539666536643939396664653437343337626332353366303366313665616361383639633030
64643834613562666134626262356330313134363239363762373362373462333435366331393466
37336134306135363832626434366434376165343932316139373765346138616535313566373832
63346530363266306333633535653233623436653031383536363133613165303566393133316434
36343765326562383364396638643436303762666166663235646166386165326163616439313933
63356565373432653165336562653833626536363261313138313532383030306363626465643662
66333439653630346534663138323437306466653164336135343338393164653334626535303163
623765666431333132653565303231373461
=======
66386163643862646634343162646465663762643034303563333833633661333932646164656462
6166646131623132393238336263623562373065643633310a663232373237396361623462613333
62373663383565623263306539636431623230633065626363666531366662363065343066363031
3738616663336665340a326138333634306137363837396366303861663064326333613662656630
62306331646637326637363766366237663037306665343761643263646663316535343561623137
63313365653535393639626465343232396261643239303430383138633135346466323834336665
33633064353034613836313265613466623961373565363835343430373138376336363966316365
35663664396436313432376264316663326130306134326231303234393561643436623039613136
63366638396262383762383336643930343661636461646162653734336334306239383132643435
39333665643738643966356431333830646561353263353063326330643731616130396466343339
39346437653063303336626663623835613938633834396430353634383366386237353862643766
37393738353231666565303031393839306463373461393761653866653330646534393832303264
30323038646166366465396235623731343539313633326539663966333437623733626131653437
62326632656462383835656235373664366566343866383938343639613737623631616231616135
633863383761366461363532353137323936
>>>>>>> main
39633436633363346633333638666438316131313337333132396634393538383432623239316463
3435333238376331383439366161656639353039326163370a376363633535623363383233646533
63363865646561323132663032383331346332373364333465643330616638623466333039623831
6530663236313234360a336264656239383166663934303335386238386139386132626165386138
32663164326237323534636263663263666634383339613362633939323565356437663666653436
31353362316334313561333430626361616337643133346664636434313664373333653839323630
65346331383135656135386263643564333063626563336365333865663333353337393866666139
64613735663363323938633161666662653161633835383832656164343836383339376661396332
66353131373265373931366130383632633466363036373562363232663162333966316563373535
65343336363732303132366335616335333334373063313562336330336661353239646533356461
62313365633336613037626261353639323937363066363062356234653631346233373965636461
63326237663537363338346566326232353632663463386135393535343436373335393430393865
33393631623762636230656263363462346561323064653561393666373735313836666238323238
66366564666266343636663666386566336637373036633966643961346636373066356632326464
63336565656666336436383938346461646431353265353133633736363761623634346262616436
61653633326333356330626638386665313865343233393637623662383634346534326537623662
34326633623431343835346339656335386330333664373166313766366339663736376261343965
63616461666230616131326537373130313239663931313330356538356161333537666237376362
64613232333834303737323438616437303666643166383439393030316533343530363863613034
39653761626439356133393164363561316535633230633438316137623333376633663665393634
63333161376263613766353030616336386531303565346263366239653232333764
21 changes: 21 additions & 0 deletions safegraph/run-safegraph.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
#
# Run the Safegraph indicator
#

set -eo pipefail

# Purge the receiving directory.
echo "Purging ./receiving..."
rm -f ./receiving/*

# Run the indicator code.
echo "Running the indicator..."
env/bin/python -m delphi_safegraph

# Copy the files to the ingestion directory.
#scp $(date +"receiving/%Y%m*") delphi.midas.cs.cmu.edu:/common/covidcast/receiving/safegraph/
#scp $(date --date='-1 month' +"receiving/%Y%m*") delphi.midas.cs.cmu.edu:/common/covidcast/receiving/safegraph/
echo "Copying files to the ingestion directory..."
cp $(date +"receiving/%Y%m*") ./test-output/safegraph 2>/dev/null # Hack to make cp care less about missing files.
cp $(date --date='-1 month' +"receiving/%Y%m*") ./test-output/safegraph