Deploying to gh-pages from @ c1c184d 🚀

_config.yml

@@ -0,0 +1,8 @@
+title: Twitter community notes
+
+defaults:
+  -
+    scope:
+      path: ""
+    values:
+      layout: "default"

_data/meta.json

@@ -0,0 +1 @@
+{"scraped_at": "2024-09-19T04:20:04.059383+00:00", "most_recent": "2024-09-17T01:01:46+00:00", "total_notes": 10310, "total_fetched": 7227, "locked_until": "2024-09-19T04:40:30+00:00", "total_tweets": 8867}

_data/mps.json

@@ -0,0 +1 @@
+[10955042, 14077382, 14395178, 14561015, 14590758, 14728535, 14845210, 15197262, 15500138, 15580900, 15726425, 15747950, 15765684, 15792947, 15865570, 15887422, 15957319, 16348332, 17390474, 17752924, 17755776, 18020612, 18099795, 18632946, 18951643, 19017675, 19126349, 19177609, 19245478, 19295262, 19407599, 19581817, 19589280, 19698191, 19859690, 19915728, 20000725, 20035457, 20132840, 20141120, 20142835, 20148039, 20226550, 20325923, 20362684, 20379080, 20392734, 20424362, 20427568, 20443985, 20508666, 20763907, 20813135, 20815162, 20924620, 20995648, 21084719, 21134627, 21189360, 21608406, 21713090, 21769986, 21868093, 22359874, 22398060, 22474050, 22645918, 22904581, 23089237, 23100225, 23105852, 23330953, 23357503, 23424533, 23749162, 23930261, 24045870, 24228290, 24727170, 24781358, 24919478, 25091518, 25587142, 26054047, 26985345, 27212119, 27286910, 27844589, 28079313, 28329646, 29746583, 29757382, 30221653, 30385952, 30978439, 31230434, 31756041, 32369044, 32439211, 32965648, 33027054, 33031863, 33038062, 33546827, 33921661, 34230896, 34374472, 36023636, 36664451, 36924726, 40297217, 40336185, 40444755, 40680478, 41195676, 41614566, 41783821, 41905997, 43312141, 45227179, 45314159, 45884485, 47561197, 49619469, 49720593, 51062977, 51174420, 52124856, 53358128, 54351001, 56338737, 56657187, 58821256, 61046587, 61660254, 61733092, 61781260, 61862447, 62093763, 63111042, 64679759, 65324089, 66078636, 67289397, 68659708, 69733004, 70803659, 71777646, 72341341, 72883916, 73923338, 77234984, 77808680, 77981490, 78126370, 78702900, 78996834, 80021045, 80151292, 80247465, 80812413, 81367969, 82093305, 87647640, 89307992, 90681210, 91786287, 93860818, 94701778, 95074968, 97932789, 98358195, 98390335, 101278778, 101712079, 102023574, 102105819, 102107018, 102696964, 103879974, 105800463, 107722321, 108055233, 108654496, 109845429, 110417782, 110550982, 111973662, 112398730, 113476721, 113491007, 114505454, 115643685, 117777690, 120720108, 124270074, 125287190, 129569947, 129768260, 130092487, 130133607, 130254822, 130280613, 130717174, 134196574, 134511026, 135019823, 136398656, 140107265, 143433549, 143508762, 144180351, 146162768, 148446462, 153810216, 153817010, 154661114, 155289056, 155698530, 164226176, 165387465, 168260255, 169241887, 169864622, 169898513, 172090468, 174386076, 174396155, 175512020, 175786071, 176721099, 176818616, 179970314, 188412761, 189280488, 191807697, 192935794, 194809611, 195752750, 198236824, 200538258, 200700960, 202558774, 202610289, 204873579, 207707329, 208516133, 209029473, 209746549, 212019800, 213243048, 215782428, 216516145, 219330249, 219976700, 222748037, 222762589, 225857392, 234089713, 235526305, 236588290, 237341484, 240202308, 240799464, 243135658, 243306810, 245849058, 247595473, 248186795, 249845912, 250601094, 255277792, 260760660, 261886643, 263691967, 264149246, 269788397, 271868768, 274033876, 276868661, 284147164, 288259199, 293147252, 293693143, 295948042, 296942850, 297967419, 298536072, 300872531, 301713110, 302302898, 302691378, 303321306, 304023718, 315283542, 316349960, 328634628, 329863756, 336529567, 342187925, 347347870, 349075382, 350223904, 350331171, 351077887, 351600865, 351657777, 351812038, 355623405, 368314502, 370700221, 376161322, 382845430, 383238347, 385306338, 389434876, 393436815, 397566701, 398132018, 398627964, 401568431, 404115169, 406646373, 407741330, 408454349, 409432990, 412213474, 415910519, 426809037, 431730103, 432396682, 443166451, 449148784, 450711709, 454933267, 456537189, 468621844, 474180693, 475138726, 477802917, 481120639, 485146783, 487655647, 493494710, 494082207, 496060628, 504523786, 515520108, 517998217, 518660800, 522273829, 532351874, 536479831, 538352656, 545081356, 546021288, 549492065, 562866860, 568249574, 570352145, 574843120, 576682647, 579804819, 580749365, 587742302, 592855211, 609096352, 620007630, 622426905, 706747004, 722347303, 728039018, 737066858, 748846627, 759067032, 794109804, 804178058, 804596460, 805185025, 808662330, 813678139, 818278134, 831250927, 860471443, 869651114, 883173469, 927480949, 948015937, 950737508, 952069440, 989608254, 990891818, 994816520, 997293914, 1086696404, 1087324843, 1114681081, 1123375357, 1151672852, 1170457166, 1179455215, 1198816026, 1302003805, 1328897358, 1337725045, 1342945016, 1364153984, 1381684742, 1444709809, 1466783923, 1513382018, 1562872584, 1573638926, 1585889305, 1608370866, 1633543440, 1678239019, 1704239983, 1704469674, 1722829178, 1858780626, 1864159236, 1865540413, 2157036506, 2159153839, 2173779986, 2174657848, 2183385501, 2187566068, 2205165174, 2209331701, 2262623615, 2278921206, 2323710210, 2350624098, 2350948635, 2382227424, 2406093795, 2425571623, 2469794059, 2471414497, 2483456906, 2491491242, 2522525965, 2528020694, 2540941937, 2583099904, 2601366250, 2609201187, 2616215901, 2718096642, 2718417153, 2750166746, 2776689260, 2785648892, 2788637049, 2797497890, 2797521996, 2805519629, 2834640796, 2855602085, 2860522038, 2872987265, 2873822342, 2933530091, 2942626846, 2979449745, 2979630241, 2979642791, 2999868527, 3005893073, 3012159189, 3021495911, 3034284694, 3056307455, 3092429597, 3128468829, 3183943337, 3298553870, 3307960979, 3323245259, 3792653536, 4010545737, 4104745522, 4397367323, 4453006828, 4897196079, 4898067429, 700732124299714561, 704931810216644609, 725725130186444800, 727494797490049024, 728595635713282048, 732224740820193280, 748521119691464705, 757652483472752640, 761499948890329088, 796764331266473984, 797021745563766785, 802599212194131968, 807297775176138752, 809357945725190144, 835388009533030401, 839428390201470976, 849799226523570177, 857641376628043778, 858747670562734081, 859085802218893312, 859339993810161664, 859433240452583424, 860188105982320644, 860807798480941056, 861333913695924225, 862712954336292864, 868787843510013953, 869291187630428160, 873636843627196416, 887252498054033408, 893178169380962304, 912263373752020993, 914410646548205568, 917084834874445824, 918729143159853056, 967847140105052165, 968505100372267009, 983085276766621696, 990637064696758272, 999264330179891200, 1043825118819110912, 1064912412736917504, 1079489837097455616, 1093899576329801729, 1111190141736046592, 1113783447561043972, 1117104681002721288, 1121375429884039175, 1133305785910550528, 1133792320040701953, 1148340474534666240, 1156498325056696320, 1158490894028460032, 1168968080690749441, 1172535926209298432, 1178829710186467328, 1179516894862364673, 1184537212303630336, 1191274965783531520, 1192899400936374272, 1192911732823482370, 1202243093552148482, 1214108358292041728, 1233337992670072837, 1244645536973676552, 1258352022132346880, 1260896506363641856, 1298994924101095425, 1303073868240875522, 1344403235742879745, 1355609944528191491, 1355938448016887808, 1393142778142527491, 1439528766368268292, 1459907434890375170, 1475756824787918851, 1483044185297068034, 1499421123594924034, 1511355740367298562, 1512840908064108545, 1518319401380024325, 1521111364399468545, 1526871150965440513, 1530711181169672194, 1530991036725608453, 1533495425524809731, 1536784150971260928, 1543007983361314821, 1544382687921799169, 1545003717388378112, 1547647624978739204, 1554589610881286145, 1568570575727075329, 1572344860052492288, 1581393252829302785, 1582194024445726720, 1582714278632689664, 1591728223292989441, 1596061977004097536, 1599753147861422087, 1602226274495303682, 1604889595056988160, 1610060002848276481, 1620056193300299778, 1620114766906773504, 1621807711510740993, 1636111129192611840, 1641121619442556928, 1655535691214778369, 1660431957849784320, 1664208792383373315, 1669648544020545538, 1692175731324047360, 1713959748284854272, 1715777573488111616, 1724049395685969920, 1727693587091996672, 1733600668412186624, 1740014998749913088, 1751336830723555328, 1754451616294408193, 1754991198886895616, 1760942562687225856, 1763897247857803264, 1768365178922409984, 1770463440437411840, 1781431480624754688, 1790468410909839360, 1795697527011864576, 1797169888831721472, 1797389916638212097, 1797567996258357248, 1797607388746297344, 1798364617116102656, 1798665613151178752, 1805309466302132224, 1810349008688422912, 1812997362262089728, 1813998834512506880]

_includes/commify.html

@@ -0,0 +1,21 @@
+{%- comment -%}
+This is a ludicrously complicated way of adding thousands separators to large
+numbers.
+
+I couldn’t figure out a better way of doing it without requiring either Ruby or
+Javascript.
+{%- endcomment -%}
+
+{%- assign number_arr = include.number | split: '' -%}
+{%- assign commified_number = '' -%}
+
+{%- for digit in number_arr reversed -%}
+  {%- assign remainder = forloop.index0 | modulo: 3 -%}
+  {%- if remainder == 0 and forloop.index0 != 0 -%}
+    {%- assign commified_number = digit | append: ',' | append: commified_number -%}
+  {%- else -%}
+    {%- assign commified_number = digit | append: commified_number -%}
+  {%- endif -%}
+{%- endfor -%}
+
+{{- commified_number -}}

_layouts/default.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>{{ page.title }}{% if page.url != '/' %} | {{ site.title }}{% endif %}</title>
+  <link href="//cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
+  <link rel="stylesheet" href="//cdn.datatables.net/v/bs5/jq-3.7.0/dt-2.0.7/fh-4.0.1/sp-2.3.1/sl-2.0.1/datatables.min.css">
+  <style type="text/css">
+    h3 {
+      margin-top: 3rem;
+    }
+    td {
+      word-break: break-word;
+    }
+    time {
+      cursor: help;
+      text-decoration: underline;
+      text-decoration-style: dotted;
+    }
+    .dt-type-numeric {
+      text-align: left !important;
+    }
+    .w-inherit {
+      width: inherit;
+    }
+  </style>
+</head>
+<body>
+  <script src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
+  <script src="//cdnjs.cloudflare.com/ajax/libs/luxon/2.3.1/luxon.min.js"></script>
+  <script src="//cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
+  <script src="//cdn.datatables.net/v/bs5/jq-3.7.0/dt-2.0.7/fh-4.0.1/sp-2.3.1/sl-2.0.1/datatables.min.js"></script>
+
+  <div class="container py-3">
+    <header class="d-flex flex-wrap justify-content-end py-3 mb-4 border-bottom">
+      <ul class="nav nav-pills">
+        <li class="nav-item">
+          <a class="nav-link{% if page.url == '/' %} active{% endif %}" href="{{ '/' | relative_url }}">Home</a>
+        </li>
+        <li class="nav-item">
+          <a class="nav-link{% if page.url == '/about/' %} active{% endif %}" href="{{ '/about/' | relative_url }}">About</a>
+        </li>
+      </ul>
+    </header>
+
+    {% if site.data.meta.total_tweets == 0 %}
+      <div class="alert alert-danger" role="alert">
+        <strong>Something’s gone wrong.</strong> There are currently no notes to show. We’re investigating.
+      </div>
+    {% elsif site.data.meta.total_fetched < site.data.meta.total_tweets %}
+      {% assign perc_unfetched = site.data.meta.total_fetched | times: -100 | divided_by: site.data.meta.total_tweets | plus:100 %}
+      <div class="alert alert-warning" role="alert">
+        <strong>Tweets are currently being indexed.</strong> The {{ perc_unfetched }}% of unindexed tweets will not be included in search or filtering. <em><a class="alert-link" href="{{ '/about/#tweet-indexing-status' | relative_url }}">More…</a></em>
+      </div>
+    {% endif %}
+
+    <main>
+      <h1>{{ page.title }}</h1>
+
+      {{ content }}
+    </main>
+  </div>
+</body>
+</html>

about/index.md

@@ -0,0 +1,73 @@
+---
+title: How it works
+---
+
+Community note data is fetched regularly [from Twitter (X)](https://x.com/i/communitynotes/download-data).
+
+This data is always a couple of days old (**most recent data is from <time class="dt" datetime="{{ site.data.meta.most_recent }}" title="{{ site.data.meta.most_recent | date_to_rfc822 }}">{{ site.data.meta.most_recent }}</time>, scraped <time class="dt" datetime="{{ site.data.meta.scraped_at }}" title="{{ site.data.meta.scraped_at | date_to_rfc822 }}">{{ site.data.meta.scraped_at }}</time>**).
+
+Notes are excluded if they meet any of the following criteria:
+
+* Created more than a week ago
+* Classifying the post as ‘not misleading’ (i.e. in support of the post)
+* Currently rated ‘unhelpful’
+
+We also attempt to filter out notes for deleted tweets and non-English tweets.
+
+---
+
+### Filter by author group
+
+With thanks to [@leobenedictus](https://x.com/leobenedictus) for the suggestion, community notes can be filtered by current UK MPs.
+
+---
+
+### Special Twitter (X) language codes
+
+When Twitter (X) can’t determine the language of a tweet, it uses one of several reserved language codes. For the purpose of language filtering, we’ve grouped these all together. But this is the breakdown:
+
+|---------------|---------------------------------------------|
+| Language code | Description                                 |
+|---------------|---------------------------------------------|
+| `art`         | Tweet contains emojis only                  |
+| `qam`         | Tweet contains mentions only                |
+| `qct`         | Tweet contains cashtags only                |
+| `qht`         | Tweet contains hashtags only                |
+| `qme`         | Tweet contains media only                   |
+| `qst`         | Tweet text is very short                    |
+| `und`         | Undefined (couldn’t determine the language) |
+| `zxx`         | Tweet contains media or twitter card only   |
+{: .table .table-striped .w-inherit }
+
+---
+
+### Tweet indexing status
+
+After fetching new proposed community notes, the text of the tweets that the notes reference is not immediately searchable. In order to make it searchable, we need to fetch these tweets – a process that can take several hours. You can see the current status below.
+
+{% if site.data.meta.total_tweets > 0 %}
+  {% assign perc_fetched = site.data.meta.total_fetched | times: 100 | divided_by: site.data.meta.total_tweets %}
+{% else %}
+  {% assign perc_fetched = 0 %}
+{% endif %}
+
+<div class="progress my-2" style="max-width: 500px;" role="progressbar">
+  <div class="progress-bar progress-bar-striped bg-{% if perc_fetched == 100 %}success{% elsif perc_fetched < 50 %}danger{% else %}warning{% endif %}" style="width: {{ perc_fetched }}%"></div>
+</div>
+{{ perc_fetched }}% of tweets ({% include commify.html number=site.data.meta.total_fetched %} / {% include commify.html number=site.data.meta.total_tweets %}) are currently searchable.
+
+<script>
+  const dts = document.getElementsByClassName('dt');
+  for (var i = 0; i < dts.length; i++) {
+    var dt = dts[i];
+    dt.textContent = luxon.DateTime.fromISO(dt.textContent).toRelative();
+  }
+</script>
+
+---
+
+### Why is the language unknown for some tweets?
+
+Until we’ve fetched a tweet, we don’t know its language. So ‘unknown language’ may mean we haven’t yet fetched that tweet. Once we’ve fetched it (in the next hour or so) we should know the tweet author, language and text.
+
+‘Unknown language’ may also mean the tweet has been deleted. In this case, we have no way of determining the tweet author, language or text.