From 7db145261bb47d58a20c9dff6e90a23669d00e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tao=20Bojl=C3=A9n?= <2803708-taobojlen@users.noreply.gitlab.com> Date: Sat, 27 Jul 2019 17:58:40 +0000 Subject: [PATCH] quantitative color coding --- CHANGELOG.md | 2 + README.md | 25 +++-- backend/config/config.exs | 2 + backend/lib/backend/api.ex | 2 +- backend/lib/backend/instance.ex | 2 + backend/lib/backend/scheduler.ex | 55 ++++++++++ backend/lib/backend_web/views/graph_view.ex | 22 +++- .../lib/backend_web/views/instance_view.ex | 3 +- .../20190727140717_add_statuses_per_day.exs | 9 ++ frontend/src/assets/nlnet.png | Bin 0 -> 4912 bytes .../src/components/atoms/FloatingCard.tsx | 1 + frontend/src/components/atoms/GraphKey.tsx | 95 ++++++++++++++---- .../src/components/molecules/Cytoscape.tsx | 29 +++++- .../src/components/molecules/GraphTools.tsx | 17 +++- frontend/src/components/organisms/Graph.tsx | 18 ++-- .../src/components/screens/AboutScreen.tsx | 29 ++++-- .../src/components/screens/InstanceScreen.tsx | 32 +++++- frontend/src/constants.tsx | 14 +++ frontend/src/redux/reducers.ts | 6 +- frontend/src/redux/types.ts | 12 ++- frontend/src/types.ts | 25 ++++- frontend/src/util.ts | 18 +++- nlnet-logo.png | Bin 0 -> 5400 bytes 23 files changed, 349 insertions(+), 69 deletions(-) create mode 100644 backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs create mode 100644 frontend/src/assets/nlnet.png create mode 100644 nlnet-logo.png diff --git a/CHANGELOG.md b/CHANGELOG.md index 3325ea3..f7499c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ElasticSearch full-text search over instance domains and descriptions. - Search results are now highlighted on the graph. - When you hover a search result, it is now highlighted on the graph. +- Instance details now show activity rate (average number of statuses posted per day). +- It's now possible to color code by activity rate. ### Changed diff --git a/README.md b/README.md index 3eddbef..8c152c9 100644 --- a/README.md +++ b/README.md @@ -59,22 +59,31 @@ If running in docker, this means you run This project doesn't crawl personal instances: the goal is to understand communities, not individuals. The threshold for what makes an instance "personal" is defined in the [backend config](backend/config/config.exs) and the [graph builder SQL](gephi/src/main/java/space/fediverse/graph/GraphBuilder.java). ## Deployment + You don't have to follow these instructions, but it's one way to set up a continuous deployment pipeline. The following are for the backend; the frontend is just a static HTML/JS site that can be deployed anywhere. + 1. Install [Dokku](http://dokku.viewdocs.io/dokku/) on your web server. 2. Install [dokku-postgres](https://github.com/dokku/dokku-postgres), [dokku-monorepo](https://github.com/notpushkin/dokku-monorepo), and [dokku-letsencrypt](https://github.com/dokku/dokku-letsencrypt). 3. Create the apps - * `dokku apps:create phoenix` - * `dokku apps:create gephi` + +- `dokku apps:create phoenix` +- `dokku apps:create gephi` + 4. Create the backing database - * `dokku postgres:create fediversedb` - * `dokku postgres:link fediversedb phoenix` - * `dokku postgres:link fediversedb gephi` + +- `dokku postgres:create fediversedb` +- `dokku postgres:link fediversedb phoenix` +- `dokku postgres:link fediversedb gephi` + 5. Update the backend configuration. In particular, change the `user_agent` in [config.exs](/backend/config/config.exs) to something descriptive. 6. Push the apps, e.g. `git push dokku@:phoenix` (note that the first push cannot be from the CD pipeline). 7. Set up SSL for the Phoenix app - * `dokku letsencrypt phoenix` - * `dokku letsencrypt:cron-job --add` + +- `dokku letsencrypt phoenix` +- `dokku letsencrypt:cron-job --add` + 8. Set up a cron job for the graph layout (use the `dokku` user). E.g. + ``` SHELL=/bin/bash 0 2 * * * /usr/bin/dokku run gephi java -Xmx1g -jar build/libs/graphBuilder.jar @@ -82,6 +91,6 @@ SHELL=/bin/bash ## Acknowledgements -[![NLnet logo](https://i.imgur.com/huV3rvo.png)](https://nlnet.nl/project/fediverse_space/) +[![NLnet logo](/nlnet-logo.png)](https://nlnet.nl/project/fediverse_space/) Many thanks to [NLnet](https://nlnet.nl/project/fediverse_space/) for their support and guidance of this project. diff --git a/backend/config/config.exs b/backend/config/config.exs index ecef9a5..18eb2c0 100644 --- a/backend/config/config.exs +++ b/backend/config/config.exs @@ -74,6 +74,8 @@ config :backend, Backend.Scheduler, {"15 0 * * *", {Backend.Scheduler, :generate_edges, []}}, # 00.30 every night {"30 0 * * *", {Backend.Scheduler, :generate_insularity_scores, []}}, + # 00.45 every night + {"45 0 * * *", {Backend.Scheduler, :generate_status_rate, []}}, # Every 3 hours {"0 */3 * * *", {Backend.Scheduler, :check_for_spam_instances, []}} ] diff --git a/backend/lib/backend/api.ex b/backend/lib/backend/api.ex index 4dcc230..ec935ce 100644 --- a/backend/lib/backend/api.ex +++ b/backend/lib/backend/api.ex @@ -37,7 +37,7 @@ defmodule Backend.Api do i.user_count >= ^user_threshold and not i.opt_out ) |> maybe_filter_nodes_to_neighborhood(domain) - |> select([c], [:domain, :user_count, :x, :y, :type]) + |> select([c], [:domain, :user_count, :x, :y, :type, :statuses_per_day]) |> Repo.all() end diff --git a/backend/lib/backend/instance.ex b/backend/lib/backend/instance.ex index e0d6891..d497cf4 100644 --- a/backend/lib/backend/instance.ex +++ b/backend/lib/backend/instance.ex @@ -10,6 +10,7 @@ defmodule Backend.Instance do field :version, :string field :insularity, :float field :type, :string + field :statuses_per_day, :float field :base_domain, :string field :opt_in, :boolean field :opt_out, :boolean @@ -39,6 +40,7 @@ defmodule Backend.Instance do :insularity, :updated_at, :type, + :statuses_per_day, :base_domain, :opt_in, :opt_out diff --git a/backend/lib/backend/scheduler.ex b/backend/lib/backend/scheduler.ex index 5ca1eaa..d2899d9 100644 --- a/backend/lib/backend/scheduler.ex +++ b/backend/lib/backend/scheduler.ex @@ -76,6 +76,61 @@ defmodule Backend.Scheduler do ) end + @doc """ + This function calculates the average number of statuses per hour over the last month. + """ + def generate_status_rate() do + now = get_now() + # We want the earliest sucessful crawl so that we can exclude it from the statistics. + # This is because the first crawl goes up to one month into the past -- this would mess up the counts! + # The statistics from here assume that all statuses were written at exactly the crawl's inserted_at timestamp. + earliest_successful_crawl_subquery = + Crawl + |> group_by([c], c.instance_domain) + |> select([c], %{ + instance_domain: c.instance_domain, + earliest_crawl: min(c.inserted_at) + }) + + instances = + Crawl + |> join(:inner, [c], c2 in subquery(earliest_successful_crawl_subquery), + on: c.instance_domain == c2.instance_domain + ) + |> where( + [c, c2], + c.inserted_at > c2.earliest_crawl and not is_nil(c.statuses_seen) and is_nil(c.error) + ) + |> select([c], %{ + instance_domain: c.instance_domain, + status_count: sum(c.statuses_seen), + second_earliest_crawl: min(c.inserted_at) + }) + |> group_by([c], c.instance_domain) + |> Repo.all() + |> Enum.map(fn %{ + instance_domain: domain, + status_count: status_count, + second_earliest_crawl: oldest_timestamp + } -> + time_diff_days = NaiveDateTime.diff(now, oldest_timestamp, :second) / (3600 * 24) + + # (we're actually only ever updating, not inserting, so inserted_at will always be ignored...) + %{ + domain: domain, + statuses_per_day: status_count / time_diff_days, + updated_at: now, + inserted_at: now + } + end) + + Instance + |> Repo.insert_all(instances, + on_conflict: {:replace, [:statuses_per_day, :updated_at]}, + conflict_target: :domain + ) + end + @doc """ This function aggregates statistics from the interactions in the database. It calculates the strength of edges between nodes. Self-edges are not generated. diff --git a/backend/lib/backend_web/views/graph_view.ex b/backend/lib/backend_web/views/graph_view.ex index 04afab0..b231177 100644 --- a/backend/lib/backend_web/views/graph_view.ex +++ b/backend/lib/backend_web/views/graph_view.ex @@ -3,9 +3,24 @@ defmodule BackendWeb.GraphView do alias BackendWeb.GraphView def render("index.json", %{nodes: nodes, edges: edges}) do + statuses_per_day = + nodes + |> Enum.map(fn %{statuses_per_day: statuses_per_day} -> statuses_per_day end) + |> Enum.filter(fn s -> s != nil end) + %{ - nodes: render_many(nodes, GraphView, "node.json", as: :node), - edges: render_many(edges, GraphView, "edge.json", as: :edge) + graph: %{ + nodes: render_many(nodes, GraphView, "node.json", as: :node), + edges: render_many(edges, GraphView, "edge.json", as: :edge) + }, + metadata: %{ + ranges: %{ + statusesPerDay: [ + Enum.min(statuses_per_day), + Enum.max(statuses_per_day) + ] + } + } } end @@ -22,7 +37,8 @@ defmodule BackendWeb.GraphView do id: node.domain, label: node.domain, size: size, - type: node.type + type: node.type, + statusesPerDay: node.statuses_per_day }, position: %{ x: node.x, diff --git a/backend/lib/backend_web/views/instance_view.ex b/backend/lib/backend_web/views/instance_view.ex index 8391062..3c04b8e 100644 --- a/backend/lib/backend_web/views/instance_view.ex +++ b/backend/lib/backend_web/views/instance_view.ex @@ -42,7 +42,8 @@ defmodule BackendWeb.InstanceView do peers: render_many(filtered_peers, InstanceView, "instance.json"), lastUpdated: last_updated, status: status, - type: instance.type + type: instance.type, + statusesPerDay: instance.statuses_per_day } end end diff --git a/backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs b/backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs new file mode 100644 index 0000000..0598d22 --- /dev/null +++ b/backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs @@ -0,0 +1,9 @@ +defmodule Backend.Repo.Migrations.AddStatusesPerHour do + use Ecto.Migration + + def change do + alter table(:instances) do + add :statuses_per_day, :float + end + end +end diff --git a/frontend/src/assets/nlnet.png b/frontend/src/assets/nlnet.png new file mode 100644 index 0000000000000000000000000000000000000000..27fad9897d532c27a92a5ddfe08974b2ad60381c GIT binary patch literal 4912 zcmV-06VL34P)002A)1^@s6;cKxK00004b3#c}2nYxW zdOV8FWQhbW?9;ba!ELWdL_~cP?peYja~^ zaAhuUa%Y?FJQ@H160J!@K~#90?VWkFRmGLYe>`SH1pS)yVzPH}GIEDDaGqA>!32rBU9k9|(R^Xl%Z zbMJc>e3198RSRC7uj=e`?!HyK_TE*ugCv#_z6u-wGy)p|eSig7nFc%oJObPWe7>xN zYR4Cg4$D9R_XYk3*tAOiyTGr3i-FloOQ?3V4J=Is91Q#l7~R75?*In__t#6HcC)Z3TzF$7S#6v zE@+K_?PzG&bY)YON)E@{8X+eMOXrri?0`|ui5=C5=l^9k91|+V} zq;lcW0XU*&>Dtj|Q7;AjCT0T`aYcl?J@7cT0-lLYwcm?yf0przIJku}wxbQAUW<^Y zXuk5D39Lq?-S;(MGSJ28<0n|eB*dA(DM4L-mhZ*L{=nfu-6Y_mFF=+r6!jPZ8-Srf z-A(w5j*0ln2XF!wBM$MRQ`cCnaBUR30lNgHp7!{1gK*l6p$oPsSREVLR?GfB7ITku z4*1VFeT)FUlDJ1lEN*;8Q2(LB9b7JQo5pab?zv?yMH@hw0ybba*N=cTo$F^~k&Q(` zUke>>k1FKpYlaQ@1=T;D%J1Av0KUGk>}RhX4-&F9Rcs7rha%2XH-Rr<9>r_V2sc zw47sWhx<0>kp5PrulImko%&(fzaIdvWW}=!mlt@`?}|G5jl*pFRSR{^%(GAdR|WnA zY?J78F7|Dm4b0E#-VHcgVLrQXiG3WFq!23T98BRqa_;vRY-(OnMu-tk-HNJ!$Ix8) zAfbRAF;{@6e2akJVJ?xko$F5kdlvp4fDOB$FYrTPM=UdXG^V640OwK% zK7to92phGF_ykL1OD)j80vw4s>*p8hdtkoPMqpjcj(!D82A)sLFalet4#qyWXS4EF zs*!`^fS(6-R{)1KtI0I+az|V5lGLdNjUOUufz!`#B{fvpM+Zqim9)g^rxwhVG@=IC z^Rk(wiINs({ro1Y?;~lvq*;;seJJUOO#W2=yE<(jPw;n;bhDafmy@#k!ICbJG(RHG zTatFK@!1cR^jxGrNqSh)hKcqKl1`Vsr}2`WaQgU5;re(ECLQ{XD0SnUcN>d^aHJBIm*o=LWrHis#{s0gt=J(U z&jSvBv81OZO^`HMm5WQs789X)_M;Zab{aApB<$Q1_R5mZMi#g$$+s(uSRz~fttzvj zLfj##OGK7f*+^#pgt({5_O~bGohNCM%v*4lq}L<7b0U3rm;Yg|l91>5ggpO{bfavx z`i0C~@bH~T-`}CG`!stH@sj{65n6<_`Cb5_!CtWUlN> z&@_9RfuAeHJwZPUl(wU)DTg`o-CjjI&wfoUEUt0*zl?B?bhxp1mKV9IglAC74C`Hof>v6a(i9oe+- zsH78A8BrM<^sz)1Rq>{byCe5KHzMzW4tGq1i)>2srm$5LZJu)Yu|>s$4tMWF-y1pn z4-uSuvTBIf{$;NVeh)$-U(}7bWS` z$PJEgxW%wj7n!f>GTEwlTBM)xk^AP@(P`7);hh}mAK4cu#2X2IS5^As7Afxbn4?Oh zLYxOV{CgApH68vt4sW!>`&_j<|8I)Ov8ThmFD7>b7Dg&!72qUiu$+xemuF&A;0)kF zEQ{sL$WVNoGYn4!*1|$EL$Ga|M`F`sECno zMP*jV$E3rBwaT-FQ}?)2=S`iRu+OUuFJRO65Mb{JXP8qL&GzmD{1LbiSgBBdEVcLf?lehR>shD)FHrhz1 z?#T!@_MxQY*~F>ywmPKbdBUlS*=`=*!Q#r3vl7O!J$TO=&EM>VZT|iP&G*~(0M5fw zs#>ybZcQx3s)!|+qV=Tt*2-a7IUCFU`%jvmiXh-0_)3<;TeN^ZTTuZ&OY~9eh*OIK zhF?Um(Ika>$m5NG8#r|fvBLmf#%8ytuxE+A*A&)w>fVg9v$=Q;`SH%{WwuVr@D0l# zNv}D5P0Fr&E`TED$-;xhK7Ufu<^V^I)g7Ma6GP@I@_c$J8*Qq?-z37F=x|eRkHHTA zQ(1Jx<9CwuNx)w$Kje9z(|4_j3#sDyi?@jJ^F1NcsoZ-Dupe*_{$?Q$y90j()(+}t z(=_ECLOS)RCziHTwCm0#KK)#Yjo=}|)G+Wz1h7=5aMRR0;x<}uDvt>HjQFV5+o?-! z2=hk2Da4O1Zlv-=v1wEuZIaiAsCJiOnVr}TODzkrdaOZlQ_?RndoaYdz^hnB^lnrd z-aW9z$z{n?C-6ym%kyl_=p*(7toYDHVZpZ;exE6`Ivv}yTtPq_j2kU zi*UUU{CVQ$$8ah?j9J7LBvcMa`y@SiQug0wPF-vZgJ)-tO7xxMAmGLl&O-b}Sm^6* zLNFvY$UONe%#pbRCP{iywuFyiu`JY<8nQb`+Drbl{h6f37EA+;s*JlnnBbq~-1}Dv z{!x+VT`R)B%Hh7RxXZ>%#rl3io}WAX1(E*8INT+Ywu$uJMbgCuZYn1b`SI&dWq!>H zeyo#mPA9cufuwJg2pifxVSG|DCw8?7t;oW7Ui>+gmUV~2KPthG?AF;$hB(FZ5^ zmj&FFVke_I=Siv_nC?gW@-~@-?UH+)ESkn3d*9-hlPsoD&=tGYE z9DW!1CQmh(r}TD*4oatyHJgcQQH(*d%!;dIsZ>|Vrk3MnN?JDJt)20&mf0*@Cb-@( zJ}km*l!b+2II~2qot-wXRPjD;bK3n~amDo#Hoc6J?>W5NN@Qt}C6R`BLGi(0r2O2; z6yB6Hv_uwUVNEYC{Zc4TEecpxzNB~#-?wE8iV&wo_+IW@Zj+rZOZRzEHg%?Am=}c) z?3Dj@+Ff78^A1vF-rI>KZGKkZE|I^$y3CQM>`R1faqvRWZvT=tz2q-=LcA|~Pql2b zU00R9(r<>m)|?Ocb!>zA`dRrBww*CNOlTBvFO~-6VK3m8pso{k0z=V#aT*qLz8Z50 zq#)qOz^NrXuv5RnNX65{eTGGJ#I}HLi@B9{#bUfEY=gy7yJ05>gxG^hzubMq)4Yo4 zj=8S3#6F8XsP1g;t+7bTrkIQA?^x93x$NJUXxY!Fb;I88F4+4SO?ZGn4H_Mma1&9? zPVtg9i-@Ivh4$RVS^dRI;|@u~Tk%}&Wf_)g&j<7nI;6uh0&|r8MqKRB@$kHv+;JNo z*_-1r;AG%|lKZuz4WTs(7%H3>Zi4bdg<_f3Vf3J1_V24$cyM~5Ov6He?O4vx8U=hm z_`55nG~s!EH(@qO47W$>!{d7AX8&HEm3rVVp1}uI(5j$83%vPNO+z@cr0LEiSdd$OY<*O6VGY0!9i*xW(_piV{ zryO@>auhKkD~~wsx4_=fYM5PmA^xFICujBNWq*pct3pf!tKJqL*v7fme1N^&ly$3cM4Z-5-?@^iG*AI)Wd$ZEPn0eC$ zJF)EStavyQIE$vFl+na;I$L2tj#MRL?u%ZMZcy_hKyFdocX+1LcfC)L>L^QLN`>g| z%1Ryz{CXzKNQe8Znl{=ai|Baq+r4E@>G?9JeXp!fZ{c*V%+-_seSzY8F~-Yc^S_a~ zR$|X|oor#~UE4|KxDUg3{bXA;zFVlfQ0ATq`&dclVtPjwnnbqm$USYNH6!4Ym_1g+ zzW5ojbFypwX>!@@iJuzO1^XF;A|A)j;%E(qU?#&4u&F4_s_BNEW%yf~Tp4-hPH3NY z#u7dAgwd6-Z~yboeTQPtaxs?PJ{&uL@#P4&n1nh5-`4JzJ;%80p5Css9F4?hq~X|S z8k$ktW6w09Q1^UR7iI-DW_4q+trY8FZZ7X#v`PU*?pGiSlU;( z9r4PR2v~xl*pz!K_6wik3+hGXUWpwr+a~0$gMD#wCE6H!x84Um6jS7Lu{^o)*cZ{8 z_Hu5HEhBxNgiXQMXZ;Sr|2d4#SngviMSd8*qV&U_ySQc;g_%X6BY7xxi0O^k!tjzz zHecymIgRy}9a0oxf@(Y3EiYbL)Mv6NOFgGwWC8rAOE}YHX(5InUt@$z(lB z7Lm9`mdN4tIaSiJ!Si0{@cLxe#>jTOjgw7}+XcM7vg!O7S&TM6a>FxO8nbJ$Oz&sp z?{FL{i(CIRd)DJ*X?R|Y{X$89%KpE7cE8xWs7okrfBCpu>2pttrMu!=FHf(7j9$4} zmd2Fgj8xP9*m*LgjeW85c=S6mo98Wu(?j+_eoakEHdmF$T;%sI=32fi|G2eNG8yx1 zpD$Az(A zGrV?PWWjeY9ZIr2%S%J~@_Sc1N)J*xohvT`WFKxfRi53Fmu_kvpH9VBJ7@1Mm5S9_ z&G(9QNc7oJ={-K7Y~DK?P{n)g81xy<=d9On#dXDw+6aFy`2S~mY>VInm;v8ADS zN0@{yRF;>+$J@~Y)bkh9dte5AObKhjW7rDR+h5R*6$thGfxJ(!tcP)!TcIUbf_c|> zrd|2|B~Z^1u!!MUIA&j}3-<|&FpR_B&EJj{6iZ71bF6{INJnCE(t((Rw*m7Ny?{-( izX#q}nu4^WZQ`Gfe$WNIQc3^-0000(); +const ColorSchemeSelect = Select.ofType(); const StyledLi = styled.li` margin-top: 2px; @@ -14,16 +16,49 @@ const StyledLi = styled.li` const StyledKeyContainer = styled.div` margin-top: 10px; `; +const ColorKeyContainer = styled.div` + display: flex; + flex-direction: row; + height: 100px; +`; +const ColorBarContainer = styled.div` + width: 10px; + display: flex; + flex-direction: column; + margin-right: 10px; +`; +interface IColorBarProps { + color: string; +} +const ColorBar = styled.div` + width: 10px; + background-color: ${props => props.color}; + flex: 1; +`; +const TextContainer = styled.div` + display: flex; + flex-direction: column; + justify-content: space-between; +`; interface IGraphKeyProps { - current?: IColorSchemeType; - colorSchemes: IColorSchemeType[]; - onItemSelect: (colorScheme?: IColorSchemeType) => void; + current?: IColorScheme; + colorSchemes: IColorScheme[]; + ranges?: { [key: string]: [number, number] }; + onItemSelect: (colorScheme?: IColorScheme) => void; } -const GraphKey: React.FC = ({ current, colorSchemes, onItemSelect }) => { +const GraphKey: React.FC = ({ current, colorSchemes, ranges, onItemSelect }) => { const unsetColorScheme = () => { onItemSelect(undefined); }; + let key; + if (current) { + if (current.type === "qualitative") { + key = renderQualitativeKey(current.values); + } else if (current.type === "quantitative") { + key = renderQuantitativeKey(ranges![current.cytoscapeDataKey]); + } + } return (
Color coding
@@ -42,27 +77,49 @@ const GraphKey: React.FC = ({ current, colorSchemes, onItemSelec />