Skip to content

Commit ba99d6a

Browse files
sgratzlsgratzl
committed
feat: work on analytics
1 parent 47be103 commit ba99d6a

11 files changed

+273
-82
lines changed

‎src/server/_analytics.py

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
from sqlalchemy import Table, Column, String, Integer, DateTime, func
2+
from ._db import metadata
3+
from flask import request
4+
from ._common import db
5+
6+
analytics_table = Table(
7+
"api_analytics",
8+
metadata,
9+
Column("id", Integer, primary_key=True, autoincrement=True),
10+
Column("datetime", DateTime),
11+
Column("ip", String(15)),
12+
Column("ua", String(1024)),
13+
Column("source", String(32)),
14+
Column("result", Integer),
15+
Column("num_rows", Integer),
16+
)
17+
18+
19+
def _derive_endpoint():
20+
endpoint = request.values.get("endpoint", request.values.get("source"))
21+
if endpoint:
22+
return endpoint
23+
path: str = request.path
24+
if path.startswith("/"):
25+
path = path[1:]
26+
if path.endswith("/"):
27+
path = path[:-1]
28+
if path == "api.php":
29+
return ""
30+
return path
31+
32+
33+
def record_analytics(result: int, num_rows=0):
34+
ip = request.headers.get(
35+
"HTTP_X_FORWARDED_FOR", request.headers.get("REMOTE_ADDR", "")
36+
)
37+
ua = request.headers.get("HTTP_USER_AGENT", "")
38+
endpoint = _derive_endpoint()
39+
40+
stmt = analytics_table.insert().values(
41+
datetime=func.now(),
42+
source=endpoint,
43+
ip=ip,
44+
ua=ua,
45+
result=result,
46+
num_rows=num_rows,
47+
)
48+
49+
db.execute(stmt)

‎src/server/_config.py

+2
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,5 @@
1010
os.environ.get("SQLALCHEMY_ENGINE_OPTIONS", "{}")
1111
)
1212
SECRET = os.environ["SECRET"]
13+
14+
AUTH = {"fluview": "xxx"}

‎src/server/_printer.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
from flask import request, jsonify, abort, make_response, request
22
from werkzeug.exceptions import HTTPException
33
from flask.json import dumps
4+
from ._analytics import record_analytics
5+
from typing import Dict
46

57
MAX_RESULTS = 1000
68

@@ -15,10 +17,10 @@ def __init__(self, message: str, status_code: int = 500):
1517
self.code = status_code if _is_using_status_codes() else 200
1618
self.response = make_response(
1719
dumps(dict(result=-1, message=message)),
18-
mimetype="application/json",
19-
status_code=self.code,
20+
self.code,
2021
)
21-
# TODO record_analytics($this->endpoint, $this->result);
22+
self.response.mimetype = "application/json"
23+
record_analytics(-1)
2224

2325

2426
class MissingOrWrongSourceException(EpiDataException):
@@ -59,7 +61,8 @@ def print_non_standard(self, data):
5961
prints a non standard JSON message
6062
"""
6163
self._result = 1
62-
# TODO record_analytics($this->endpoint, $this->result);
64+
record_analytics(1)
65+
# TODO
6366
return jsonify(dict(result=self._result, message="success", epidata=data))
6467

6568
@property

‎src/server/_query.py

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
from ._db import metadata
2+
from typing import Optional, List, Union, Tuple
3+
from ._validate import DateRange
4+
5+
def date_string(value: int) -> str:
6+
# converts a date integer (YYYYMMDD) into a date string (YYYY-MM-DD)
7+
# $value: the date as an 8-digit integer
8+
year = int(value / 10000) % 10000
9+
month = int(value / 100) % 100
10+
day = value % 100
11+
return "{0:04d}-{1:02d}-{2:%02d}".format(year, month, day)
12+
13+
14+
def filter_dates(field: str, dates: Optional[List[DateRange]]) -> str:
15+
"""
16+
builds a SQL expression to filter values/ranges of dates
17+
:param field: name of the field to filter
18+
:param data: array of date values/ranges
19+
"""
20+
if not dates:
21+
return None
22+
23+
$filter = null;
24+
foreach($dates as $date) {
25+
if($filter === null) {
26+
$filter = '';
27+
} else {
28+
$filter .= ' OR ';
29+
}
30+
if(is_array($date)) {
31+
// range of values
32+
$first = date_string($date[0]);
33+
$last = date_string($date[1]);
34+
$filter .= "({$field} BETWEEN '{$first}' AND '{$last}')";
35+
} else {
36+
// single value
37+
$date = date_string($date);
38+
$filter .= "({$field} = '{$date}')";
39+
}
40+
}
41+
return $filter;
42+
}
43+
44+
// builds a SQL expression to filter values/ranges of integers (ex: epiweeks)
45+
// $field: name of the field to filter
46+
// $values: array of integer values/ranges
47+
function filter_integers($field, $values) {
48+
$filter = null;
49+
foreach($values as $value) {
50+
if($filter === null) {
51+
$filter = '';
52+
} else {
53+
$filter .= ' OR ';
54+
}
55+
if(is_array($value)) {
56+
// range of values
57+
$filter .= "({$field} BETWEEN {$value[0]} AND {$value[1]})";
58+
} else {
59+
// single value
60+
$filter .= "({$field} = {$value})";
61+
}
62+
}
63+
return $filter;
64+
}
65+
66+
// builds a SQL expression to filter strings (ex: locations)
67+
// $field: name of the field to filter
68+
// $values: array of values
69+
function filter_strings($field, $values) {
70+
global $dbh;
71+
$filter = null;
72+
foreach($values as $value) {
73+
if($filter === null) {
74+
$filter = '';
75+
} else {
76+
$filter .= ' OR ';
77+
}
78+
if(is_array($value)) {
79+
// range of values
80+
$value0 = mysqli_real_escape_string($dbh, $value[0]);
81+
$value1 = mysqli_real_escape_string($dbh, $value[1]);
82+
$filter .= "({$field} BETWEEN '{$value0}' AND '{$value1}')";
83+
} else {
84+
// single value
85+
$value = mysqli_real_escape_string($dbh, $value);
86+
$filter .= "({$field} = '{$value}')";
87+
}
88+
}
89+
return $filter;
90+
}

‎src/server/_validate.py

+43-62
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,76 @@
11
from flask import request
2-
from typing import Iterable, List, Union, Optional
3-
from ._printer import APrinter
2+
from typing import Iterable, List, Union, Optional, Tuple
3+
from ._printer import ValidationFailedException
44

55

6-
def require_all(printer: APrinter, *values: Iterable[str]) -> bool:
6+
def require_all(*values: Iterable[str]) -> bool:
77
"""
8-
returns true if all fields are present in the request
8+
returns true if all fields are present in the request otherwise raises an exception
99
:returns bool
1010
"""
1111
for value in values:
1212
if not request.values.get(value):
13-
printer.print_validation_failed(f"missing parameter: need [{values}]")
14-
return False
13+
raise ValidationFailedException(f"missing parameter: need [{values}]")
1514
return True
1615

1716

18-
def require_any(printer: APrinter, *values: Iterable[str]) -> bool:
17+
def require_any(*values: Iterable[str]) -> bool:
1918
"""
20-
returns true if any fields are present in the request
19+
returns true if any fields are present in the request otherwise raises an exception
2120
:returns bool
2221
"""
2322
for value in values:
2423
if request.values.get(value):
2524
return True
26-
printer.print_validation_failed(f"missing parameter: need one of [{values}]")
27-
return False
25+
raise ValidationFailedException(f"missing parameter: need one of [{values}]")
2826

2927

30-
def date_string(value: int) -> str:
31-
# converts a date integer (YYYYMMDD) into a date string (YYYY-MM-DD)
32-
# $value: the date as an 8-digit integer
33-
year = int(value / 10000) % 10000
34-
month = int(value / 100) % 100
35-
day = value % 100
36-
return "{0:04d}-{1:02d}-{2:%02d}".format(year, month, day)
28+
def extract_strings(key: str) -> Optional[List[str]]:
29+
s = request.values.get(key)
30+
if not s:
31+
# nothing to do
32+
return None
33+
return s.split(",")
3734

3835

39-
def extract_values(
40-
s: str, value_type="str"
41-
) -> Optional[List[Union[Tuple[str, str], Tuple[int, int], str, int]]]:
42-
# extracts an array of values and/or ranges from a string
43-
# $str: the string to parse
44-
# $type:
45-
# - 'int': interpret dashes as ranges, cast values to integers
46-
# - 'ordered_string': interpret dashes as ranges, keep values as strings
47-
# - otherwise: ignore dashes, keep values as strings
48-
if not str:
36+
IntRange = Union[Tuple[int, int], int]
37+
38+
39+
def extract_ints(key: str) -> Optional[List[IntRange]]:
40+
s = request.values.get(key)
41+
if not s:
4942
# nothing to do
5043
return None
51-
# whether to parse a value with a dash as a range of values
52-
should_parse_range = value_type == "int" or value_type == "ordered_string"
53-
# maintain a list of values and/or ranges
54-
values: List[Union[Tuple[str, str], Tuple[int, int], str, int]] = []
55-
# split on commas and loop over each entry, which could be either a single value or a range of values
56-
parts = s.split(",")
57-
for part in parts:
58-
if should_parse_range and "-" in part:
59-
# split on the dash
60-
[first, last] = part.split("-", 2)
61-
if value_type == "int":
62-
first = int(first)
63-
last = int(last)
64-
if first == last:
65-
# the first and last numbers are the same, just treat it as a singe value
66-
values.append(first)
67-
elif last > first:
68-
# add the range as an array
69-
values.append((first, last))
70-
else:
71-
# the range is inverted, this is an error
72-
return None
73-
else:
74-
# this is a single value
75-
if value_type == "int":
76-
# cast to integer
77-
value = int(part)
78-
else:
79-
# interpret the string literally
80-
value = part
81-
# add the extracted value to the list
82-
values.append(value)
83-
# success, return the list
84-
return values
44+
45+
def _parse_range(part: str):
46+
if "-" not in part:
47+
return int(part)
48+
r = part.split("-", 2)
49+
first = int(r[0])
50+
last = int(r[1])
51+
if first == last:
52+
# the first and last numbers are the same, just treat it as a singe value
53+
return first
54+
elif last > first:
55+
# add the range as an array
56+
return (first, last)
57+
# the range is inverted, this is an error
58+
return None
59+
60+
values = [_parse_range(part) for part in s.split(",")]
61+
# check for invalid values
62+
return None if any(v is None for v in values) else values
8563

8664

8765
def parse_date(s: str) -> int:
8866
# parses a given string in format YYYYMMDD or YYYY-MM-DD to a number in the form YYYYMMDD
8967
return int(s.replace("-", ""))
9068

9169

92-
def extract_dates(s: str) -> Optional[List[Union[Tuple[int, int], int]]]:
70+
DateRange = Union[Tuple[int, int], int]
71+
72+
73+
def extract_dates(s: str) -> Optional[List[DateRange]]:
9374
# extracts an array of values and/or ranges from a string
9475
# $str: the string to parse
9576
if not s:

‎src/server/api_helpers.php

-7
Original file line numberDiff line numberDiff line change
@@ -115,13 +115,6 @@ function get_region_states($region) {
115115
return null;
116116
}
117117

118-
function record_analytics($source, $result, $num_rows = 0) {
119-
global $dbh;
120-
$ip = mysqli_real_escape_string($dbh, isset($_SERVER['REMOTE_ADDR']) ? $_SERVER['REMOTE_ADDR'] : '');
121-
$ua = mysqli_real_escape_string($dbh, isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
122-
$source = mysqli_real_escape_string($dbh, isset($source) ? $source : '');
123-
mysqli_query($dbh, "INSERT INTO `api_analytics` (`datetime`, `ip`, `ua`, `source`, `result`, `num_rows`) VALUES (now(), '{$ip}', '{$ua}', '{$source}', {$result}, {$num_rows})");
124-
}
125118

126119
// executes a query, casts the results, and returns an array of the data
127120
// the number of results is limited to $MAX_RESULTS

‎src/server/endpoints/__init__.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from . import covidcast_meta, flusurv, fluview
2+
3+
endpoints = [covidcast_meta, flusurv, fluview]
4+
5+
__all__ = ["endpoints"]

‎src/server/endpoints/covidcast_meta.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from sqlalchemy import select
44
from .._common import app, db
55

6-
bp = Blueprint("covidcast_meta", __name__, url_prefix="/covidcast_meta")
6+
bp = Blueprint("covidcast_meta", __name__)
77

88

99
@bp.route("/", methods=("GET", "POST"))

0 commit comments

Comments
 (0)