Testing

Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3-alpine
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+CMD ["python", "stats.py"]

config.ini

@@ -0,0 +1,11 @@
+[db]
+host = 127.0.0.1
+user = user
+pass = pass
+db = db
+
+[fetch]
+enable_file_output = false
+output_path = /data
+team_id = 0
+interval = 15

requirements.txt

@@ -0,0 +1,6 @@
+configparser
+PyMySQL
+requests
+pytz
+mysql-connector-python
+python-dateutil

stats.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+import configparser
+import datetime
+import json
+import os
+import pathlib
+import sys
+
+import mysql.connector
+import requests
+import distutils
+
+import dateutil.parser
+import pytz
+import tables
+import time
+
+
+## Configuration
+config = configparser.ConfigParser()
+config.read_file('config.ini')
+
+MYSQL_HOST: str = os.environ.get('MYSQL_HOST') or config['db'].get('host')
+MYSQL_USER: str = os.environ.get('MYSQL_USER') or config['db'].get('user')
+MYSQL_PASS: str = os.environ.get('MYSQL_PASS') or config['db'].get('pass')
+MYSQL_DB: str = os.environ.get('MYSQL_DB') or config['db'].get('db')
+FILE_OUTPUT: bool = distutils.util.strtobool(os.environ.get('FETCH_ENABLE_FILE_OUTPUT') or config['fetch'].getboolean('enable_file_output') or "no")
+OUTPUT_PATH: str = os.environ.get('FETCH_OUTPUT_PATH') or config['fetch'].get('output_path')
+FAH_TEAM: int = int(os.environ.get('FETCH_TEAM_ID')) or config['fetch'].getint('team_id')
+""" Interval in minutes. If unset, script runs once only. """
+FETCH_INTERVAL: int = int(os.environ.get('FETCH_INTERVAL')) or config['fetch'].getint('interval', fallback=None)
+
+def init_db(cursor: mysql.connector.cursor.MySQLCursor) -> None:
+    cursor.execute(table.TABLE_STATEMENT)
+    cursor.execute('SET SESSION time_zone="Etc/UTC";')
+
+## Business
+def business():
+    http_data = requests.get('https://stats.foldingathome.org/api/team/{}'.format(FAH_TEAM))
+    data = http_data.json()
+    time = datetime.datetime.utcnow()
+
+    if FILE_OUTPUT:
+        fn = os.path.join(OUTPUT_PATH, str(FAH_TEAM), time.isoformat())
+        with open(fn, 'w') as f:
+            f.write(http_data.text)
+    
+    conn: mysql.connector.MySQLConnection = mysql.connector.connect(
+        host=MYSQL_HOST,
+        user=MYSQL_USER,
+        password=MYSQL_PASS,
+        db=MYSQL_DB
+    )
+
+    try:
+        cursor = conn.cursor(prepared=True)
+        init_db(cursor)
+        team_sql = 'INSERT IGNORE INTO `team` (`team_id`, `name`) VALUES (?, ?)'
+        team_data = (data['team'], data['name'])
+        cursor.execute(team_sql, team_data)
+        cursor.execute(
+            '''INSERT INTO team_stats (wus, team_rank, total_teams,
+            active_50days, last_update, team_id, credit, fetched_at)
+            VALUES (?,?,?,?,?,?,?,?)''',
+            (data['wus'], data['rank'], data['total_teams'],
+            data['active_50'], data['last'], data['team'], data['credit'], time)
+        )
+        donor_data = []
+        donor_stats = []
+        for donor in data['donors']:
+            donor_data.append((donor['id'], donor['name']))
+            donor_stats.append((
+                donor['id'], donor['wus'], donor.get('rank'), donor['credit'],
+                donor['team'], time
+            ))
+        cursor.executemany(
+            'INSERT IGNORE INTO donor (user_id, name) VALUES (?,?)',
+            donor_data
+        )
+        cursor.executemany(
+            'INSERT INTO donor_stats (donor_id, wus, donor_rank, credit,'
+            'team_id, fetched_at) VALUES (?,?,?,?,?,?)',
+            donor_stats
+        )
+        conn.commit()
+        cursor.close()
+        conn.close()
+    except:
+        raise
+
+def main():
+    if FETCH_INTERVAL is None:
+        business()
+    else:
+        while True:
+            business()
+            time.sleep(FETCH_INTERVAL * 60)
+            

tables.py

@@ -0,0 +1,61 @@
+TABLE_STATEMENT='''create table if not exists donor
+(
+    user_id int  not null,
+    name    text null,
+    constraint donor_user_id_uindex
+        unique (user_id)
+);
+
+alter table donor
+    add primary key (user_id);
+
+create table if not exists team
+(
+    team_id int  not null,
+    name    text null,
+    constraint team_team_id_uindex
+        unique (team_id)
+);
+
+alter table team
+    add primary key (team_id);
+
+create table if not exists donor_stats
+(
+    id         int auto_increment
+        primary key,
+    donor_id   int      null,
+    wus        int      null,
+    donor_rank int      null,
+    credit     int      null,
+    team_id    int      null,
+    fetched_at datetime null,
+    constraint donor_stats_donor_user_id_fk
+        foreign key (donor_id) references donor (user_id),
+    constraint donor_stats_team_team_id_fk
+        foreign key (team_id) references team (team_id)
+);
+
+create index if not exists donor_stats_fetched_at_index
+    on donor_stats (fetched_at);
+
+create table if not exists team_stats
+(
+    id            int      not null
+        primary key,
+    wus           int      null,
+    team_rank     int      null,
+    total_teams   int      null,
+    active_50days int      null,
+    last_update   datetime null,
+    team_id       int      null,
+    credit        int      null,
+    fetched_at    datetime null,
+    constraint team_stats_team_team_id_fk
+        foreign key (team_id) references team (team_id)
+);
+
+create index if not exists team_stats_fetched_at_index
+    on team_stats (fetched_at);
+
+'''