chore: snapshot production code before Apple Design UI refinement
This commit is contained in:
lingniu
@@ -0,0 +1,49 @@
|
||||
"""Consistent logical backup of the complete current MySQL database; no DB writes."""
|
||||
import gzip, hashlib, json, os, pathlib, time
|
||||
import pymysql
|
||||
from db_access import connect
|
||||
os.umask(0o077)
|
||||
root=pathlib.Path('/opt/lingniu-go-native/backups/mileage-reconciliation-20260916')
|
||||
root.mkdir(parents=True,exist_ok=True)
|
||||
path=root/'database.sql.gz'
|
||||
if path.exists(): raise RuntimeError('Backup already exists; do not overwrite')
|
||||
c=connect(); manifest={'started_at':time.strftime('%Y-%m-%dT%H:%M:%S%z'),'tables':{}}
|
||||
with c.cursor() as q:
|
||||
q.execute('SELECT DATABASE() AS db'); db=q.fetchone()['db'];manifest['database']=db
|
||||
q.execute('SHOW FULL TABLES'); entries=q.fetchall()
|
||||
tables=[list(r.values())[0] for r in entries if list(r.values())[1]=='BASE TABLE']
|
||||
q.execute('SHOW TRIGGERS'); triggers=q.fetchall()
|
||||
q.execute('START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY')
|
||||
def literal(v):
|
||||
if isinstance(v,bytes): return '0x'+v.hex()
|
||||
return c.escape(v)
|
||||
with gzip.open(str(path)+'.partial','wt',encoding='utf-8',compresslevel=3) as f:
|
||||
f.write('SET NAMES utf8mb4;\nSET FOREIGN_KEY_CHECKS=0;\n')
|
||||
for table in tables:
|
||||
q.execute('SHOW CREATE TABLE `'+table+'`');ddl=list(q.fetchone().values())[1]
|
||||
f.write('DROP TABLE IF EXISTS `'+table+'`;\n'+ddl+';\n')
|
||||
q.execute('SHOW COLUMNS FROM `'+table+'`');cols=[r['Field'] for r in q.fetchall() if 'GENERATED' not in r['Extra']]
|
||||
names=','.join('`'+v+'`' for v in cols);count=0
|
||||
with c.cursor(pymysql.cursors.SSCursor) as stream:
|
||||
stream.execute('SELECT '+names+' FROM `'+table+'`')
|
||||
while True:
|
||||
rows=stream.fetchmany(500)
|
||||
if not rows:break
|
||||
f.write('INSERT INTO `'+table+'` ('+names+') VALUES\n'+',\n'.join('('+','.join(literal(v) for v in row)+')' for row in rows)+';\n')
|
||||
count+=len(rows)
|
||||
manifest['tables'][table]=count
|
||||
print(json.dumps({'table':table,'rows':count}),flush=True)
|
||||
for row in entries:
|
||||
if list(row.values())[1]=='VIEW':
|
||||
name=list(row.values())[0];q.execute('SHOW CREATE VIEW `'+name+'`');ddl=q.fetchone()['Create View'];f.write(ddl+';\n')
|
||||
for trigger in triggers:
|
||||
q.execute('SHOW CREATE TRIGGER `'+trigger['Trigger']+'`');ddl=q.fetchone()['SQL Original Statement'];f.write('DELIMITER ;;\n'+ddl+';;\nDELIMITER ;\n')
|
||||
f.write('SET FOREIGN_KEY_CHECKS=1;\n')
|
||||
c.rollback();c.close()
|
||||
os.rename(str(path)+'.partial',path)
|
||||
h=hashlib.sha256()
|
||||
with path.open('rb') as f:
|
||||
for block in iter(lambda:f.read(1024*1024),b''):h.update(block)
|
||||
manifest.update(sha256=h.hexdigest(),bytes=path.stat().st_size,finished_at=time.strftime('%Y-%m-%dT%H:%M:%S%z'))
|
||||
(root/'manifest.json').write_text(json.dumps(manifest,indent=2))
|
||||
print(json.dumps({'complete':True,'bytes':manifest['bytes'],'sha256':manifest['sha256']}),flush=True)
|
||||
@@ -0,0 +1,17 @@
|
||||
import pymysql,pathlib,re,json
|
||||
|
||||
def connect():
|
||||
env={}
|
||||
for name in ['base.env','stat-writer.env']:
|
||||
for line in pathlib.Path('/opt/lingniu-go-native/env/'+name).read_text().splitlines():
|
||||
if '=' in line and not line.startswith('#'):
|
||||
k,v=line.split('=',1);env[k]=v.strip().strip(chr(34)).strip(chr(39))
|
||||
m=re.fullmatch(r'([^:]+):(.*?)@tcp\(([^:]+):(\d+)\)/([^?]+).*',env['MYSQL_DSN'])
|
||||
return pymysql.connect(user=m[1],password=m[2],host=m[3],port=int(m[4]),database=m[5],charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor,autocommit=False)
|
||||
if __name__=='__main__':
|
||||
c=connect()
|
||||
with c.cursor() as q:
|
||||
q.execute('START TRANSACTION READ ONLY')
|
||||
for sql in ["SELECT DATABASE() AS db, VERSION() AS version", "SELECT table_name,engine,table_rows,ROUND((data_length+index_length)/1024/1024,1) AS mb FROM information_schema.tables WHERE table_schema=DATABASE() ORDER BY data_length DESC", "SELECT protocol,source_ip,COUNT(*) AS n,COUNT(DISTINCT vin) AS vins FROM vehicle_daily_mileage_source WHERE source_ip IN ('legacy-mysql.lingniu-prod','manual-lingniu-prod-day-mileage') GROUP BY protocol,source_ip"]:
|
||||
q.execute(sql);print(json.dumps(q.fetchall(),default=str))
|
||||
c.rollback();c.close()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Repair verified mileage provenance and offline baselines. Dry-run by default."""
|
||||
import argparse,datetime,json,pathlib
|
||||
from decimal import Decimal
|
||||
from db_access import connect
|
||||
p=argparse.ArgumentParser();p.add_argument('--apply',action='store_true');args=p.parse_args()
|
||||
root=pathlib.Path('/opt/lingniu-go-native/backups/mileage-reconciliation-20260916')
|
||||
legacy="('legacy-mysql.lingniu-prod','manual-lingniu-prod-day-mileage')"
|
||||
c=connect();report={}
|
||||
with c.cursor() as q:
|
||||
q.execute('START TRANSACTION READ ONLY')
|
||||
q.execute("SELECT COUNT(*) AS n,COUNT(DISTINCT vin) AS vehicles FROM vehicle_daily_mileage_source WHERE protocol='GB32960' AND source_ip IN "+legacy);report['legacy']=q.fetchone()
|
||||
q.execute("SELECT COUNT(*) AS n FROM vehicle_daily_mileage m JOIN vehicle_data_source d ON d.id=m.source_id WHERE m.protocol='GB32960' AND d.source_ip IN "+legacy);report['legacy_projections']=q.fetchone()
|
||||
q.execute("""SELECT s.vin,s.stat_date,s.protocol,s.source_key,s.latest_total_mileage_km,s.first_total_mileage_km,s.daily_mileage_km,s.first_event_time,s.latest_event_time,
|
||||
p.latest_total_mileage_km AS previous_total,p.latest_event_time AS previous_time
|
||||
FROM vehicle_daily_mileage_source s
|
||||
JOIN vehicle_daily_mileage_source p ON p.vin=s.vin AND p.protocol=s.protocol AND p.source_key=s.source_key
|
||||
AND p.stat_date=(SELECT MAX(b.stat_date) FROM vehicle_daily_mileage_source b
|
||||
WHERE b.vin=s.vin AND b.protocol=s.protocol AND b.source_key=s.source_key AND b.stat_date<s.stat_date
|
||||
AND b.quality_status='OK' AND b.latest_total_mileage_km>0
|
||||
AND b.latest_event_time>=TIMESTAMP(b.stat_date) AND b.latest_event_time<TIMESTAMP(b.stat_date)+INTERVAL 1 DAY)
|
||||
WHERE s.stat_date<='2026-09-16' AND s.quality_status='OK'
|
||||
AND s.quality_reason='current_day_first_baseline'
|
||||
AND s.latest_total_mileage_km>0 AND s.first_event_time>=TIMESTAMP(s.stat_date)
|
||||
AND s.source_ip NOT IN """+legacy+" ORDER BY s.vin,s.protocol,s.source_key,s.stat_date")
|
||||
changes=q.fetchall();report['baseline_candidates']=len(changes)
|
||||
q.execute("SELECT id FROM vehicle_data_source WHERE source_ip IN "+legacy);legacy_ids=[r['id'] for r in q.fetchall()]
|
||||
c.rollback()
|
||||
if args.apply:
|
||||
assert (root/'manifest.json').is_file() and (root/'database.sql.gz').is_file(),'Complete backup required'
|
||||
(root/'baseline-before.json').write_text(json.dumps(changes,default=str,ensure_ascii=False))
|
||||
q.execute('START TRANSACTION')
|
||||
q.execute("UPDATE vehicle_daily_mileage_source SET quality_status='INVALID_DELTA',quality_reason='UNVERIFIED_LEGACY_PROTOCOL',is_selected=0 WHERE protocol='GB32960' AND source_ip IN "+legacy)
|
||||
report['quarantined_source_rows']=q.rowcount
|
||||
q.execute("UPDATE vehicle_data_source SET enabled=0,remark='Protocol provenance unverified; quarantined 2026-09-16; original records retained' WHERE protocol='GB32960' AND source_ip IN "+legacy)
|
||||
q.execute("DELETE m FROM vehicle_daily_mileage m JOIN vehicle_data_source d ON d.id=m.source_id WHERE m.protocol='GB32960' AND d.source_ip IN "+legacy)
|
||||
report['removed_mislabelled_projections']=q.rowcount
|
||||
affected=set()
|
||||
for s in changes:
|
||||
delta=s['latest_total_mileage_km']-s['previous_total']
|
||||
days=max(1,(s['latest_event_time'].date()-s['previous_time'].date()).days)
|
||||
status='OK';reason='historical_source_baseline'
|
||||
if Decimal('-1')<=delta<0:delta=Decimal(0);reason='negative_jitter_clamped'
|
||||
elif delta<0:status='INVALID_DELTA';reason='TOTAL_MILEAGE_ROLLBACK'
|
||||
elif delta>Decimal(2500*days):status='INVALID_DELTA';reason='outside_daily_range'
|
||||
q.execute("""UPDATE vehicle_daily_mileage_source SET first_total_mileage_km=%s,first_event_time=%s,daily_mileage_km=%s,
|
||||
quality_status=%s,quality_reason=%s WHERE vin=%s AND stat_date=%s AND protocol=%s AND source_key=%s
|
||||
AND first_total_mileage_km <=> %s AND daily_mileage_km=%s""",(s['previous_total'],s['previous_time'],delta,status,reason,s['vin'],s['stat_date'],s['protocol'],s['source_key'],s['first_total_mileage_km'],s['daily_mileage_km']))
|
||||
if q.rowcount:affected.add((s['vin'],s['stat_date'],s['protocol']))
|
||||
# Restore valid non-import GB32960 projections that were hidden by migrated rows.
|
||||
q.execute("""SELECT DISTINCT s.vin,s.stat_date,s.protocol FROM vehicle_daily_mileage_source s
|
||||
JOIN vehicle_daily_mileage_source bad ON bad.vin=s.vin AND bad.stat_date=s.stat_date AND bad.protocol=s.protocol
|
||||
WHERE bad.source_ip IN """+legacy+" AND s.source_ip NOT IN "+legacy+" AND s.quality_status='OK' AND s.latest_total_mileage_km IS NOT NULL")
|
||||
affected.update((r['vin'],r['stat_date'],r['protocol']) for r in q.fetchall())
|
||||
report['reproject_days']=len(affected)
|
||||
# The Go projection utility is run before resuming writers, using canonical SQL.
|
||||
(root/'reproject.json').write_text(json.dumps([{'vin':v,'date':str(d),'protocol':p} for v,d,p in sorted(affected)]))
|
||||
c.commit()
|
||||
(root if args.apply else pathlib.Path('/tmp')).joinpath('repair-result.json' if args.apply else 'mileage-repair-dry-run.json').write_text(json.dumps(report,default=str,indent=2))
|
||||
print(json.dumps(report,default=str),flush=True)
|
||||
c.close()
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Run a specified maintenance binary without exposing environment credentials."""
|
||||
import os,pathlib,subprocess,sys
|
||||
for name in ['base.env','stat-writer.env']:
|
||||
for line in pathlib.Path('/opt/lingniu-go-native/env/'+name).read_text().splitlines():
|
||||
if '=' in line and not line.startswith('#'):
|
||||
k,v=line.split('=',1);os.environ[k]=v.strip().strip(chr(34)).strip(chr(39))
|
||||
sys.exit(subprocess.call(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user