#!/usr/bin/env python3
"""Download every file attached in your Twist workspaces.

Covers threads, comments and conversations in the channels you've joined.
Files are saved as:

    <out>/<workspace>/Channels/<channel>/<thread>/<file>
    <out>/<workspace>/Conversations/<conversation>/<file>

Usage:
    export TWIST_TOKEN=...   # the Twist API v3 token you already use
    python3 twist_attachments_backup.py --dry-run   # list files only
    python3 twist_attachments_backup.py --out twist-backup

Safe to stop (Ctrl+C) and re-run: files already saved are skipped. Anything
that failed is listed in <out>/failed.txt. Needs Python 3.9+ and nothing else.
"""

import argparse
import hashlib
import http.client
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path

API = os.environ.get("TWIST_API_URL", "https://api.twist.com/api/v3/")
TOKEN = os.environ.get("TWIST_TOKEN", "")
PAGE = 500
FILE_HOSTS = {"files.twist.com", "dnp4pehkvoo6n.cloudfront.net"}
FILE_URL = re.compile(r"https://(?:files\.twist\.com|dnp4pehkvoo6n\.cloudfront\.net)(/[^\s\[\]<>\"'?#]+)")
NETWORK_ERRORS = (OSError, http.client.HTTPException)  # includes timeouts and dropped connections


class TwistError(Exception):
    pass


def retry(fn):
    """Call fn, retrying rate limits (HTTP 429), server errors and network drops."""
    for wait in (2, 4, 8, 16, 32, 60, 120, 0):
        try:
            return fn()
        except urllib.error.HTTPError as e:
            if not wait or (e.code != 429 and e.code < 500):
                raise
            reason = f"HTTP {e.code}"
        except NETWORK_ERRORS as e:
            if not wait:
                raise
            reason = str(e) or type(e).__name__
        print(f"    {reason}, retrying in {wait}s", file=sys.stderr)
        time.sleep(wait)


def api(method, **params):
    url = API + method + "?" + urllib.parse.urlencode(params)
    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {TOKEN}"})

    def call():
        with urllib.request.urlopen(req, timeout=60) as resp:
            return json.load(resp)

    try:
        return retry(call)
    except urllib.error.HTTPError as e:
        body = e.read().decode(errors="replace")
        try:
            body = json.loads(body)["error_string"]
        except (ValueError, KeyError, TypeError):
            body = body[:300]
        error = f"Twist API {method} failed with HTTP {e.code}: {body}"
        if e.code == 401 or body == "Invalid token":
            sys.exit(error + "\nCheck that TWIST_TOKEN is a valid Twist API token.")
        raise TwistError(error)
    except NETWORK_ERRORS as e:
        sys.exit(f"Couldn't reach the Twist API ({e}). Check your connection, then run again.")


def paginate(method, cursor_param, cursor_key, **params):
    """Yield every item, oldest first. The cursor is exclusive and obj_index
    starts at 0, hence -1. Pages may come back newest-first, so take the max."""
    cursor = -1
    while True:
        page = api(method, order_by="ASC", limit=PAGE, **{cursor_param: cursor}, **params)
        yield from sorted(page, key=lambda o: o[cursor_key])
        if len(page) < PAGE:
            return
        cursor = max(o[cursor_key] for o in page)


def file_links(post):
    """Yield (path, file_name) for files attached to, or linked in, a post."""
    for a in post.get("attachments") or []:
        url = urllib.parse.urlsplit(a.get("url") or "")
        if url.hostname in FILE_HOSTS:
            yield url.path, a.get("file_name") or a.get("title")
    for m in FILE_URL.finditer(post.get("content") or ""):
        path = m[1].rstrip(".,;:!")
        while path.endswith(")") and path.count(")") > path.count("("):  # "(see <link>)"
            path = path[:-1].rstrip(".,;:!")
        yield path, None


def safe(name, max_bytes=120):
    """Make name usable as a file or folder name on Windows, macOS and Linux."""
    name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "_", name).strip()
    name = name.encode()[:max_bytes].decode(errors="ignore")
    return name.strip().rstrip(".") or "untitled"


def local_name(path, name):
    """Readable, and unique per upload, so a re-run can trust an existing file."""
    stem, ext = os.path.splitext(name or urllib.parse.unquote(path.rsplit("/", 1)[-1]))
    tag = hashlib.md5(path.encode()).hexdigest()[:8]
    ext = re.sub(r"[^\w.-]", "", ext)[:16]
    return f"{safe(stem)} ({tag}){ext}"


def download(path, dest):
    # files.twist.com only hands out the signed file link (valid for a minute)
    # to requests from the Twist apps, which it recognises by the Referer.
    url = "https://files.twist.com" + urllib.parse.quote(path, safe="/%:@!$&'()*+,;=~")
    req = urllib.request.Request(url, headers={"Referer": "https://twist.com/"})

    def attempt():
        tmp = dest.parent / f".{uuid.uuid4().hex}.part"
        try:
            with open(tmp, "xb") as f, urllib.request.urlopen(req, timeout=300) as resp:
                if urllib.parse.urlsplit(resp.geturl()).path == "/login":
                    raise RuntimeError("Twist asked for a login instead of sending the file")
                size = 0
                while chunk := resp.read(1 << 20):
                    size += f.write(chunk)
                expected = resp.headers.get("Content-Length")
                if expected and size != int(expected):
                    raise ConnectionError(f"download cut off at {size} of {expected} bytes")
            os.replace(tmp, dest)
        finally:
            tmp.unlink(missing_ok=True)

    retry(attempt)


def fail(args, stats, what, source, error):
    stats["failed"] += 1
    print(f"    FAILED: {error}", file=sys.stderr)
    if not args.dry_run:
        with open(args.out / "failed.txt", "a", encoding="utf-8") as f:
            f.write(f"{what}\t{source}\t{error}\n")


def save(folder, links, source, args, stats):
    files = {}
    for path, name in links:
        files[path] = files.get(path) or name
    for path, name in files.items():
        dest = folder / local_name(path, name)
        if dest.exists():
            stats["skipped"] += 1
            continue
        print(f"  {dest.relative_to(args.out)}")
        if args.dry_run:
            stats["found"] += 1
            continue
        try:
            folder.mkdir(parents=True, exist_ok=True)
            download(path, dest)
            stats["downloaded"] += 1
        except Exception as e:
            fail(args, stats, f"https://files.twist.com{path}", source, e)


def main():
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--out", type=Path, default=Path("twist-backup"), help="output folder")
    p.add_argument("--workspace-id", type=int, help="only this workspace")
    p.add_argument("--dry-run", action="store_true", help="list files without downloading")
    args = p.parse_args()
    if not TOKEN:
        sys.exit("Set TWIST_TOKEN to your Twist API token first.")

    if not args.dry_run:
        args.out.mkdir(parents=True, exist_ok=True)
        (args.out / "failed.txt").unlink(missing_ok=True)
    stats = dict(found=0, downloaded=0, skipped=0, failed=0)

    workspaces = [ws for ws in api("workspaces/get") if args.workspace_id in (None, ws["id"])]
    if not workspaces:
        sys.exit("No matching workspace found for this token.")

    for ws in workspaces:
        ws_dir = args.out / safe(f"{ws['name']} ({ws['id']})")
        print(f"Workspace: {ws['name']}")

        for ch in api("channels/get", workspace_id=ws["id"]):
            print(f" #{ch['name']}")
            for th in paginate("threads/get", "after_id", "id", channel_id=ch["id"]):
                source = f"https://twist.com/a/{ws['id']}/ch/{ch['id']}/t/{th['id']}/"
                try:
                    links = list(file_links(th))
                    if th.get("comment_count"):
                        for c in paginate("comments/get", "newer_than_obj_index", "obj_index", thread_id=th["id"]):
                            links += file_links(c)
                except TwistError as e:
                    fail(args, stats, "(thread)", source, e)
                    continue
                folder = ws_dir / "Channels" / safe(ch["name"]) / safe(f"{th['id']} {th['title']}")
                save(folder, links, source, args, stats)

        print(" Conversations")
        for conv in paginate("conversations/get", "after_id", "id", workspace_id=ws["id"]):
            source = f"https://twist.com/a/{ws['id']}/msg/{conv['id']}/"
            try:
                links = []
                for m in paginate("conversation_messages/get", "newer_than_obj_index", "obj_index", conversation_id=conv["id"]):
                    links += file_links(m)
            except TwistError as e:
                fail(args, stats, "(conversation)", source, e)
                continue
            folder = ws_dir / "Conversations" / safe(f"{conv['id']} {conv.get('title') or ''}")
            save(folder, links, source, args, stats)

    if args.dry_run:
        print(f"\nWould download {stats['found']} files ({stats['skipped']} already saved).")
    else:
        print(f"\nDownloaded {stats['downloaded']}, skipped {stats['skipped']} already saved, "
              f"{stats['failed']} failed.")
    if stats["failed"]:
        if not args.dry_run:
            print(f"See {args.out / 'failed.txt'}, then run again to retry.")
        sys.exit(1)


if __name__ == "__main__":
    try:
        main()
    except TwistError as e:
        sys.exit(str(e))
    except KeyboardInterrupt:
        sys.exit("\nStopped. Run again to pick up where you left off.")
