issue_comments: 590592581
This data as json
| html_url | issue_url | id | node_id | user | created_at | updated_at | author_association | body | reactions | issue | performed_via_github_app |
|---|---|---|---|---|---|---|---|---|---|---|---|
| https://github.com/simonw/datasette/pull/683#issuecomment-590592581 | https://api.github.com/repos/simonw/datasette/issues/683 | 590592581 | MDEyOklzc3VlQ29tbWVudDU5MDU5MjU4MQ== | 9599 | 2020-02-24T23:00:44Z | 2020-02-24T23:01:09Z | OWNER | I've been testing this out by running one-off demo plugins. I saved the following in a file called `write-plugins/log_asgi.py` (it's a hacked around copy of [asgi-log-to-sqlite](https://github.com/simonw/asgi-log-to-sqlite)) and then running `datasette data.db --plugins-dir=write-plugins/`: ```python from datasette import hookimpl import sqlite_utils import time class AsgiLogToSqliteViaWriteQueue: lookup_columns = ( "path", "user_agent", "referer", "accept_language", "content_type", "query_string", ) def __init__(self, app, db): self.app = app self.db = db self._tables_ensured = False async def ensure_tables(self): def _ensure_tables(conn): db = sqlite_utils.Database(conn) for column in self.lookup_columns: table = "{}s".format(column) if not db[table].exists(): db[table].create({"id": int, "name": str}, pk="id") if "requests" not in db.table_names(): db["requests"].create( { "start": float, "method": str, "path": int, "query_string": int, "user_agent": int, "referer": int, "accept_language": int, "http_status": int, "content_type": int, "client_ip": str, "duration": float, "body_size": int, }, foreign_keys=self.lookup_columns, ) await self.db.execute_write_fn(_ensure_tables) async def __call__(self, scope, receive, send): if not self._tables_ensured: self._tables_ensured = True await self.ensure_tables() response_headers = [] body_size = 0 http_status = None async def wrapped_send(message): nonlocal body_size, response_headers, http_status if message["type"] == "http.response.start": response_headers = message["headers"] http_status = message["status"] if message["type"] == "http.response.body": body_size += len(message["body"]) await send(message) start = time.time() await self.app(scope, receive, wrapped_send) end = time.time() path = str(scope["path"]) query_string = None if scope.get("query_string"): query_string = "?{}".format(scope["query_string"].decode("utf8")) request_headers = dict(scope.get("headers") or []) referer = header(request_headers, "referer") user_agent = header(request_headers, "user-agent") accept_language = header(request_headers, "accept-language") content_type = header(dict(response_headers), "content-type") def _log_to_database(conn): db = sqlite_utils.Database(conn) db["requests"].insert( { "start": start, "method": scope["method"], "path": lookup(db, "paths", path), "query_string": lookup(db, "query_strings", query_string), "user_agent": lookup(db, "user_agents", user_agent), "referer": lookup(db, "referers", referer), "accept_language": lookup(db, "accept_languages", accept_language), "http_status": http_status, "content_type": lookup(db, "content_types", content_type), "client_ip": scope.get("client", (None, None))[0], "duration": end - start, "body_size": body_size, }, alter=True, foreign_keys=self.lookup_columns, ) await self.db.execute_write_fn(_log_to_database) def header(d, name): return d.get(name.encode("utf8"), b"").decode("utf8") or None def lookup(db, table, value): return db[table].lookup({"name": value}) if value else None @hookimpl def asgi_wrapper(datasette): def wrap_with_class(app): return AsgiLogToSqliteViaWriteQueue( app, next(iter(datasette.databases.values())) ) return wrap_with_class ``` | {"total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0} | 570101428 |