#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ smcl.py -- Python replacement for smcl.php 1.0.3 (CFB, with help from WWG) Emulates ssc.php, producing Stata SMCL rather than HTML: lists SSC packages created or revised in the past month, read from the RePEc bocodex.rdf and bocodey.rdf template files. Requires only the Python 3 standard library (works with the platform python3 3.6 on RHEL 8 and 3.9+ on RHEL 9). Usage: smcl.py write SMCL to standard output smcl.py -o FILE write SMCL to FILE atomically (for cron); FILE is left untouched if the RePEc files cannot be fetched smcl.py --cgi emit a Content-Type header first (for use as CGI) Exit status is 0 on success, 1 if a source file could not be fetched. cfb 20261025: remove bocodey """ import os import re import sys import time import argparse import tempfile import urllib.request SOURCES = [ "http://RePEc.org/boc/bocode/bocodex.rdf", # "http://RePEc.org/boc/bocode/bocodey.rdf", # "http://RePEc.org/boc/bocode/bocodez.rdf", # disabled, as in the PHP ] TIMEZONE = "America/Detroit" FETCH_TIMEOUT = 60 # seconds per file # The PHP regexes, anchored the same way: with no multiline flag, "$" matches # at the end of the line or just before its trailing newline, and "." does # not match a newline -- identical semantics in PHP PCRE and Python re. RE_TITLE = re.compile(r"^Title: (.+): Stata (.+)$") RE_AUTHOR = re.compile(r"^Author-Name: (.+)$") RE_CREATED = re.compile(r"^Creation-Date: (.+)$") RE_REVISED = re.compile(r"^Revision-Date: (.+)$") RE_REQUIRES = re.compile(r"^Requires: (.+)$") RE_HANDLE = re.compile(r"^Handle: (.+)bocode:S(.+)\n$") _NUMERIC = re.compile(r"^\s*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?\s*$") _LEADING_NUM = re.compile(r"^\s*[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?") def php_gt(a, b): """PHP 8 '>' between two strings: numeric if both are numeric strings, otherwise a byte-wise string comparison.""" if _NUMERIC.match(a) and _NUMERIC.match(b): return float(a) > float(b) return a > b def php_num(s): """Value PHP arithmetic gives a string: its leading number, else 0.""" m = _LEADING_NUM.match(s) return float(m.group(0)) if m else 0.0 def fmt(d8): """yyyymmdd -> yyyy-mm-dd, exactly as fmt() in the PHP.""" import math v = php_num(d8) y = math.floor(v / 10000) m = math.floor(v / 100 - y * 100) d = math.floor(v - y * 10000 - m * 100) return "%d-%s%d-%s%d" % (y, "0" if m < 10 else "", m, "0" if d < 10 else "", d) def fetch_lines(url): """Like PHP file(): split on \\n, keeping the newline on each line. Decoded as Latin-1 so every byte round-trips unchanged to the output.""" with urllib.request.urlopen(url, timeout=FETCH_TIMEOUT) as resp: data = resp.read().decode("latin-1") parts = data.split("\n") lines = [p + "\n" for p in parts[:-1]] if parts[-1]: lines.append(parts[-1]) return lines def build_smcl(lines, now): out = [] w = out.append year, mon, mday = now.tm_year, now.tm_mon, now.tm_mday yrb4, monb4, dyb4 = year, mon - 1, mday if mon == 1: monb4, yrb4 = 12, year - 1 curmo = mon cmp_ = "0" if curmo < 10 else "" mp = "0" if monb4 < 10 else "" dp = "0" if dyb4 < 10 else "" w('{smcl}\n\n{txt}{net "from http://repec.org/bocode":SSC} ' 'Stata modules created or revised ') prevdt = "%d%s%d%s%d" % (yrb4, mp, monb4, dp, dyb4) w("%d-%s%d-%s%d to %d-%s%d-%s%d\n{hline}\n" % (yrb4, mp, monb4, dp, dyb4, year, cmp_, curmo, dp, dyb4)) ncre = nrev = ntot = 0 name = namelc = first = desc = an = cred = revd = cre = rev = req = "" for line in lines: m = RE_TITLE.match(line) if m: name = m.group(1) namelc = name.lower() first = name[:1].lower() desc = m.group(2) an = "" req = "" # see note in README: PHP let this carry over ntot += 1 m = RE_AUTHOR.match(line) if m: an = an + m.group(1).rstrip(" \t\n\r\0\x0b") + " " m = RE_CREATED.match(line) if m: cred = m.group(1) cre = fmt(cred) m = RE_REVISED.match(line) if m: revd = m.group(1) rev = fmt(revd) m = RE_REQUIRES.match(line) if m: req = m.group(1) m = RE_HANDLE.match(line) if m: if php_gt(cred, prevdt) or php_gt(revd, prevdt): w('\n{net "describe http://repec.org/bocode/%s/%s":%s}\n' % (first, namelc, name)) w("{txt} %s" % desc) w("{txt} Authors: %s Req: %s\n" % (an, req)) if php_gt(revd, cred): w("{txt} Revised: %s\n" % rev) nrev += 1 else: w("{txt} Created: %s\n" % cre) ncre += 1 cred = revd = an = "" w("\n{hline}\n{txt} End of recent additions and updates\n") w("{txt} Packages Created: %d \n" % ncre) w("{txt} Packages Revised: %d \n" % nrev) w("{txt} Total Packages: %d \n" % ntot) return "".join(out) def main(): ap = argparse.ArgumentParser(description="SMCL listing of recent SSC packages") ap.add_argument("-o", "--output", help="write to this file (atomic replace)") ap.add_argument("--cgi", action="store_true", help="emit a CGI header") args = ap.parse_args() os.environ["TZ"] = TIMEZONE time.tzset() now = time.localtime() lines = [] try: for url in SOURCES: lines.extend(fetch_lines(url)) except Exception as e: sys.stderr.write("smcl.py: cannot fetch %s: %s\n" % (url, e)) if args.cgi: sys.stdout.write("Status: 503 Service Unavailable\r\n" "Content-Type: text/plain\r\n\r\n" "RePEc listing temporarily unavailable\n") return 1 body = build_smcl(lines, now).encode("latin-1") if args.output: target = os.path.abspath(args.output) fd, tmp = tempfile.mkstemp(dir=os.path.dirname(target), prefix=".smcl.") try: with os.fdopen(fd, "wb") as f: f.write(body) os.chmod(tmp, 0o644) os.replace(tmp, target) except Exception: try: os.unlink(tmp) except OSError: pass raise else: out = sys.stdout.buffer if args.cgi: out.write(b"Content-Type: text/plain; charset=ISO-8859-1\r\n\r\n") out.write(body) out.flush() return 0 if __name__ == "__main__": sys.exit(main())