Color code graph

This commit is contained in:
Tao Bojlén 2019-07-24 15:51:44 +00:00
parent 2de412bfbb
commit 3b515c3a36
25 changed files with 353 additions and 68 deletions

View file

@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- It is now possible to color code the graph by instance type (e.g. Mastodon, Pleroma, etc.)
### Changed
### Deprecated

View file

@ -35,7 +35,7 @@ defmodule Backend.Api do
i.user_count >= ^user_threshold
)
|> maybe_filter_nodes_to_neighborhood(domain)
|> select([c], [:domain, :user_count, :x, :y])
|> select([c], [:domain, :user_count, :x, :y, :type])
|> Repo.all()
end

View file

@ -14,6 +14,8 @@ defmodule Backend.Crawler.ApiCrawler do
# {domain_mentioned, count}
@type instance_interactions :: %{String.t() => integer}
@type instance_type :: :mastodon | :pleroma | :gab
defstruct [
:version,
:description,
@ -21,7 +23,8 @@ defmodule Backend.Crawler.ApiCrawler do
:status_count,
:peers,
:interactions,
:statuses_seen
:statuses_seen,
:instance_type
]
@type t() :: %__MODULE__{
@ -31,7 +34,8 @@ defmodule Backend.Crawler.ApiCrawler do
status_count: integer,
peers: [String.t()],
interactions: instance_interactions,
statuses_seen: integer
statuses_seen: integer,
instance_type: instance_type
}
@doc """

View file

@ -103,6 +103,12 @@ defmodule Backend.Crawler do
}) do
now = get_now()
instance_type =
case result.instance_type do
nil -> nil
not_nil_type -> Atom.to_string(not_nil_type)
end
## Update the instance we crawled ##
Repo.insert!(
%Instance{
@ -110,7 +116,8 @@ defmodule Backend.Crawler do
description: result.description,
version: result.version,
user_count: result.user_count,
status_count: result.status_count
status_count: result.status_count,
type: instance_type
},
on_conflict: [
set: [
@ -118,6 +125,7 @@ defmodule Backend.Crawler do
version: result.version,
user_count: result.user_count,
status_count: result.status_count,
type: instance_type,
updated_at: now
]
],
@ -140,7 +148,7 @@ defmodule Backend.Crawler do
result.interactions
|> Map.keys()
|> list_union(result.peers)
|> Enum.filter(fn domain -> not is_blacklisted?(domain) end)
|> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end)
peers =
peers_domains

View file

@ -38,12 +38,17 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
crawl_large_instance(domain, instance)
else
Map.merge(
Map.merge(
Map.take(instance, ["version", "description"]),
Map.take(instance["stats"], ["user_count", "status_count"])
)
Map.take(instance["stats"], ["user_count"])
|> Map.new(fn {k, v} -> {String.to_atom(k), v} end),
%{peers: [], interactions: %{}, statuses_seen: 0}
%{
peers: [],
interactions: %{},
statuses_seen: 0,
instance_type: nil,
description: nil,
version: nil,
status_count: nil
}
)
end
end
@ -68,13 +73,25 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
} mentions in #{statuses_seen} statuses."
)
instance_type =
cond do
Map.get(instance, "version") |> String.downcase() =~ "pleroma" -> :pleroma
is_gab?(instance) -> :gab
true -> :mastodon
end
Map.merge(
Map.merge(
Map.take(instance, ["version", "description"]),
Map.take(instance["stats"], ["user_count", "status_count"])
)
|> Map.new(fn {k, v} -> {String.to_atom(k), v} end),
%{peers: peers, interactions: interactions, statuses_seen: statuses_seen}
%{
peers: peers,
interactions: interactions,
statuses_seen: statuses_seen,
instance_type: instance_type
}
)
end
@ -208,4 +225,18 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
Map.merge(acc, map)
end)
end
defp is_gab?(instance) do
title_is_gab = Map.get(instance, "title") |> String.downcase() == "gab social"
contact_account = Map.get(instance, "contact_account")
if contact_account != nil do
gab_keys = ["is_pro", "is_verified", "is_donor", "is_investor"]
has_gab_keys = gab_keys |> Enum.any?(&Map.has_key?(contact_account, &1))
title_is_gab or has_gab_keys
else
title_is_gab
end
end
end

View file

@ -47,6 +47,7 @@ defmodule Backend.Crawler.StaleInstanceManager do
Process.send_after(self(), :queue_stale_domains, 60_000)
end
# TODO: crawl instances with a blocking robots.txt less often (daily?)
defp queue_stale_domains() do
interval = -1 * get_config(:crawl_interval_mins)

View file

@ -9,6 +9,7 @@ defmodule Backend.Instance do
field :status_count, :integer
field :version, :string
field :insularity, :float
field :type, :string
many_to_many :peers, Backend.Instance,
join_through: Backend.InstancePeer,
@ -33,7 +34,8 @@ defmodule Backend.Instance do
:status_count,
:version,
:insularity,
:updated_at
:updated_at,
:type
])
|> validate_required([:domain])
|> put_assoc(:peers, attrs.peers)

View file

@ -4,12 +4,12 @@ defmodule BackendWeb.GraphView do
def render("index.json", %{nodes: nodes, edges: edges}) do
%{
nodes: render_many(nodes, GraphView, "node.json"),
edges: render_many(edges, GraphView, "edge.json")
nodes: render_many(nodes, GraphView, "node.json", as: :node),
edges: render_many(edges, GraphView, "edge.json", as: :edge)
}
end
def render("node.json", %{graph: node}) do
def render("node.json", %{node: node}) do
size =
case node.user_count > 1 do
true -> :math.log(node.user_count)
@ -21,7 +21,8 @@ defmodule BackendWeb.GraphView do
data: %{
id: node.domain,
label: node.domain,
size: size
size: size,
type: node.type
},
position: %{
x: node.x,
@ -30,7 +31,7 @@ defmodule BackendWeb.GraphView do
}
end
def render("edge.json", %{graph: edge}) do
def render("edge.json", %{edge: edge}) do
%{
data: %{
id: edge.id,

View file

@ -23,7 +23,8 @@ defmodule BackendWeb.SearchView do
%{
name: instance.domain,
description: description,
userCount: instance.user_count
userCount: instance.user_count,
type: instance.type
}
end
end

View file

@ -0,0 +1,9 @@
defmodule Backend.Repo.Migrations.AddInstanceType do
use Ecto.Migration
def change do
alter table(:instances) do
add :type, :string
end
end
end

View file

@ -2,13 +2,18 @@ import { Card, Elevation, ICardProps } from "@blueprintjs/core";
import * as React from "react";
import styled from "styled-components";
const FloatingCardRow = styled.div`
display: flex;
`;
const FloatingCardElement = styled(Card)`
position: absolute;
bottom: 10px;
left: 10px;
margin: 0 0 10px 10px;
z-index: 20;
`;
const FloatingCard: React.FC<ICardProps> = props => <FloatingCardElement elevation={Elevation.TWO} {...props} />;
const FloatingCard: React.FC<ICardProps> = props => (
<FloatingCardRow>
<FloatingCardElement elevation={Elevation.ONE} {...props} />
</FloatingCardRow>
);
export default FloatingCard;

View file

@ -0,0 +1,74 @@
import { Button, Classes, H5, H6, Icon, MenuItem } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import { ItemRenderer, Select } from "@blueprintjs/select";
import React from "react";
import styled from "styled-components";
import { FloatingCard } from ".";
import { QUALITATIVE_COLOR_SCHEME } from "../../constants";
import { IColorSchemeType } from "../../types";
import { capitalize } from "../../util";
const ColorSchemeSelect = Select.ofType<IColorSchemeType>();
const StyledLi = styled.li`
margin-top: 2px;
`;
const StyledIcon = styled(Icon)`
margin-right: 5px;
`;
const StyledKeyContainer = styled.div`
margin-top: 10px;
`;
interface IGraphKeyProps {
current?: IColorSchemeType;
colorSchemes: IColorSchemeType[];
onItemSelect: (colorScheme?: IColorSchemeType) => void;
}
const GraphKey: React.FC<IGraphKeyProps> = ({ current, colorSchemes, onItemSelect }) => {
const unsetColorScheme = () => {
onItemSelect(undefined);
};
return (
<FloatingCard>
<H5>Color coding</H5>
<ColorSchemeSelect
activeItem={current}
filterable={false}
items={colorSchemes}
itemRenderer={renderItem}
onItemSelect={onItemSelect}
popoverProps={{ minimal: true }}
>
<Button
text={(current && current.name) || "Select..."}
icon={IconNames.TINT}
rightIcon={IconNames.CARET_DOWN}
/>
<Button icon={IconNames.SMALL_CROSS} minimal={true} onClick={unsetColorScheme} disabled={!current} />
</ColorSchemeSelect>
{current && (
<StyledKeyContainer>
<H6>Key</H6>
<ul className={Classes.LIST_UNSTYLED}>
{current.values.map((v, idx) => (
<StyledLi>
<StyledIcon icon={IconNames.FULL_CIRCLE} color={QUALITATIVE_COLOR_SCHEME[idx]} />
{capitalize(v)}
</StyledLi>
))}
</ul>
</StyledKeyContainer>
)}
</FloatingCard>
);
};
const renderItem: ItemRenderer<IColorSchemeType> = (colorScheme, { handleClick, modifiers }) => {
if (!modifiers.matchesPredicate) {
return null;
}
return <MenuItem active={modifiers.active} key={colorScheme.name} onClick={handleClick} text={colorScheme.name} />;
};
export default GraphKey;

View file

@ -0,0 +1,13 @@
import { Button } from "@blueprintjs/core";
import * as React from "react";
import FloatingCard from "./FloatingCard";
interface IGraphResetButtonProps {
onClick: () => void;
}
const GraphResetButton: React.FC<IGraphResetButtonProps> = ({ onClick }) => (
<FloatingCard>
<Button icon="compass" title="Reset graph view" onClick={onClick} />
</FloatingCard>
);
export default GraphResetButton;

View file

@ -1,2 +1,4 @@
export { default as Page } from "./Page";
export { default as FloatingCard } from "./FloatingCard";
export { default as GraphKey } from "./GraphKey";
export { default as GraphResetButton } from "./GraphResetButton";

View file

@ -3,7 +3,8 @@ import * as React from "react";
import ReactDOM from "react-dom";
import styled from "styled-components";
import tippy, { Instance } from "tippy.js";
import { DEFAULT_NODE_COLOR, SELECTED_NODE_COLOR } from "../../constants";
import { DEFAULT_NODE_COLOR, QUALITATIVE_COLOR_SCHEME, SELECTED_NODE_COLOR } from "../../constants";
import { IColorSchemeType } from "../../types";
const CytoscapeContainer = styled.div`
width: 100%;
@ -12,20 +13,15 @@ const CytoscapeContainer = styled.div`
`;
interface ICytoscapeProps {
colorScheme?: IColorSchemeType;
currentNodeId: string | null;
elements: cytoscape.ElementsDefinition;
navigateToInstancePath?: (domain: string) => void;
navigateToRoot?: () => void;
}
class Cytoscape extends React.Component<ICytoscapeProps> {
class Cytoscape extends React.PureComponent<ICytoscapeProps> {
private cy?: cytoscape.Core;
public shouldComponentUpdate(prevProps: ICytoscapeProps) {
// We only want to update this component if the current instance selection changes.
// We know that the `elements` prop will never change so we skip the expensive computations here.
return prevProps.currentNodeId !== this.props.currentNodeId;
}
public componentDidMount() {
const container = ReactDOM.findDOMNode(this);
this.cy = cytoscape({
@ -63,24 +59,8 @@ class Cytoscape extends React.Component<ICytoscapeProps> {
});
const style = this.cy.style() as any;
style
.clear()
.selector("node")
.style({
"background-color": DEFAULT_NODE_COLOR,
// The size from the backend is log_10(userCount), which from 10 <= userCount <= 1,000,000 gives us the range
// 1-6. We map this to the range of sizes we want.
// TODO: I should probably check that that the backend is actually using log_10 and not log_e, but it look
// quite good as it is, so...
height: "mapData(size, 1, 6, 20, 200)",
label: "data(id)",
width: "mapData(size, 1, 6, 20, 200)"
})
.selector("node:selected")
.style({
"background-color": SELECTED_NODE_COLOR
})
.selector("edge")
.style({
"curve-style": "haystack", // fast edges
@ -100,8 +80,8 @@ class Cytoscape extends React.Component<ICytoscapeProps> {
.selector(".thickEdge") // when a node is selected, make edges thicker so you can actually see them
.style({
width: 2
})
.update();
});
this.resetNodeColorScheme(style); // this function also called `update()`
this.cy.nodes().on("select", e => {
const instanceId = e.target.data("id");
@ -145,6 +125,9 @@ class Cytoscape extends React.Component<ICytoscapeProps> {
public componentDidUpdate(prevProps: ICytoscapeProps) {
this.setNodeSelection(prevProps.currentNodeId);
if (prevProps.colorScheme !== this.props.colorScheme) {
this.updateColorScheme();
}
}
public componentWillUnmount() {
@ -195,6 +178,54 @@ class Cytoscape extends React.Component<ICytoscapeProps> {
this.cy.center(selected);
}
};
/**
* Set the color scheme to the default. A style object can optionally be passed.
* This is used during initilization to avoid having to do multiple renderings of the graph.
*/
private resetNodeColorScheme = (style?: any) => {
if (!style) {
style = this.cy!.style() as any;
}
style
.selector("node")
.style({
"background-color": DEFAULT_NODE_COLOR,
// The size from the backend is log_10(userCount), which from 10 <= userCount <= 1,000,000 gives us the range
// 1-6. We map this to the range of sizes we want.
// TODO: I should probably check that that the backend is actually using log_10 and not log_e, but it look
// quite good as it is, so...
height: "mapData(size, 1, 6, 20, 200)",
label: "data(id)",
width: "mapData(size, 1, 6, 20, 200)"
})
.selector("node:selected")
.style({
"background-color": SELECTED_NODE_COLOR
})
.update();
};
private updateColorScheme = () => {
if (!this.cy) {
throw new Error("Expected cytoscape, but there wasn't one!");
}
const { colorScheme } = this.props;
let style = this.cy.style() as any;
if (!colorScheme) {
this.resetNodeColorScheme();
} else {
colorScheme.values.forEach((v, idx) => {
style = style.selector(`node[${colorScheme.cytoscapeDataKey} = '${v}']`).style({
"background-color": QUALITATIVE_COLOR_SCHEME[idx]
});
});
style
.selector("node:selected")
.style({ "background-color": SELECTED_NODE_COLOR })
.update();
}
};
}
export default Cytoscape;

View file

@ -1,13 +0,0 @@
import { Button } from "@blueprintjs/core";
import * as React from "react";
import { FloatingCard } from "../atoms/";
interface IFloatingResetButtonProps {
onClick?: () => any;
}
const FloatingResetButton: React.FC<IFloatingResetButtonProps> = ({ onClick }) => (
<FloatingCard>
<Button icon="compass" title="Reset graph view" onClick={onClick} />
</FloatingCard>
);
export default FloatingResetButton;

View file

@ -0,0 +1,33 @@
import React from "react";
import styled from "styled-components";
import { IColorSchemeType } from "../../types";
import { GraphKey, GraphResetButton } from "../atoms";
const GraphToolsContainer = styled.div`
position: absolute;
bottom: 0;
left: 0;
display: flex;
flex-direction: column;
`;
interface IGraphToolsProps {
currentColorScheme?: IColorSchemeType;
colorSchemes: IColorSchemeType[];
onColorSchemeSelect: (colorScheme?: IColorSchemeType) => void;
onResetButtonClick: () => void;
}
const GraphTools: React.FC<IGraphToolsProps> = ({
currentColorScheme,
colorSchemes,
onColorSchemeSelect,
onResetButtonClick
}) => {
return (
<GraphToolsContainer>
<GraphResetButton onClick={onResetButtonClick} />
<GraphKey current={currentColorScheme} colorSchemes={colorSchemes} onItemSelect={onColorSchemeSelect} />
</GraphToolsContainer>
);
};
export default GraphTools;

View file

@ -1,10 +1,14 @@
import { Card, Classes, Elevation, H4 } from "@blueprintjs/core";
import { Card, Classes, Elevation, H4, Icon } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import inflection from "inflection";
import * as numeral from "numeral";
import React from "react";
import sanitize from "sanitize-html";
import styled from "styled-components";
import { QUALITATIVE_COLOR_SCHEME } from "../../constants";
import { ISearchResultInstance } from "../../redux/types";
import { typeColorScheme } from "../../types";
import { capitalize } from "../../util";
const StyledCard = styled(Card)`
width: 80%;
@ -12,9 +16,19 @@ const StyledCard = styled(Card)`
background-color: #394b59 !important;
text-align: left;
`;
const StyledH4 = styled(H4)`
const StyledHeadingContainer = styled.div`
display: flex;
align-items: center;
margin-bottom: 5px;
`;
const StyledH4 = styled(H4)`
margin: 0 5px 0 0;
flex: 1;
`;
const StyledType = styled.div`
margin: 0;
align-self: flex-end;
`;
const StyledUserCount = styled.div`
margin: 0;
`;
@ -33,9 +47,24 @@ const SearchResult: React.FC<ISearchResultProps> = ({ result, onClick }) => {
shortenedDescription = shortenedDescription.substring(0, 100) + "...";
}
}
let typeIcon;
if (result.type) {
const idx = typeColorScheme.values.indexOf(result.type);
typeIcon = (
<StyledType className={Classes.TEXT_MUTED}>
<Icon icon={IconNames.SYMBOL_CIRCLE} color={QUALITATIVE_COLOR_SCHEME[idx]} />
{" " + capitalize(result.type)}
</StyledType>
);
}
return (
<StyledCard elevation={Elevation.ONE} interactive={true} key={result.name} onClick={onClick}>
<StyledHeadingContainer>
<StyledH4>{result.name}</StyledH4>
{typeIcon}
</StyledHeadingContainer>
{result.userCount && (
<StyledUserCount className={Classes.TEXT_MUTED}>
{numeral.default(result.userCount).format("0,0")} {inflection.inflect("people", result.userCount, "person")}

View file

@ -1,4 +1,4 @@
export { default as Cytoscape } from "./Cytoscape";
export { default as ErrorState } from "./ErrorState";
export { default as FloatingResetButton } from "./FloatingResetButton";
export { default as SearchResult } from "./SearchResult";
export { default as GraphTools } from "./GraphTools";

View file

@ -7,8 +7,9 @@ import { Dispatch } from "redux";
import styled from "styled-components";
import { fetchGraph } from "../../redux/actions";
import { IAppState, IGraph } from "../../redux/types";
import { colorSchemes, IColorSchemeType } from "../../types";
import { domainMatchSelector } from "../../util";
import { Cytoscape, ErrorState, FloatingResetButton } from "../molecules/";
import { Cytoscape, ErrorState, GraphTools } from "../molecules/";
const GraphDiv = styled.div`
flex: 2;
@ -22,12 +23,16 @@ interface IGraphProps {
isLoadingGraph: boolean;
navigate: (path: string) => void;
}
class GraphImpl extends React.Component<IGraphProps> {
interface IGraphState {
colorScheme?: IColorSchemeType;
}
class GraphImpl extends React.PureComponent<IGraphProps, IGraphState> {
private cytoscapeComponent: React.RefObject<Cytoscape>;
public constructor(props: IGraphProps) {
super(props);
this.cytoscapeComponent = React.createRef();
this.state = { colorScheme: undefined };
}
public componentDidMount() {
@ -44,13 +49,19 @@ class GraphImpl extends React.Component<IGraphProps> {
content = (
<>
<Cytoscape
colorScheme={this.state.colorScheme}
currentNodeId={this.props.currentInstanceName}
elements={this.props.graph}
navigateToInstancePath={this.navigateToInstancePath}
navigateToRoot={this.navigateToRoot}
ref={this.cytoscapeComponent}
/>
<FloatingResetButton onClick={this.resetGraphPosition} />
<GraphTools
onResetButtonClick={this.resetGraphPosition}
currentColorScheme={this.state.colorScheme}
colorSchemes={colorSchemes}
onColorSchemeSelect={this.setColorScheme}
/>
</>
);
}
@ -70,6 +81,10 @@ class GraphImpl extends React.Component<IGraphProps> {
}
};
private setColorScheme = (colorScheme?: IColorSchemeType) => {
this.setState({ colorScheme });
};
private navigateToInstancePath = (domain: string) => {
this.props.navigate(`/instance/${domain}`);
};

View file

@ -144,6 +144,8 @@ class InstanceScreenImpl extends React.PureComponent<IInstanceScreenProps, IInst
}
private processEdgesToFindNeighbors = () => {
// TODO: use cytoscape to replace this method
// simply cy.$id(nodeId).outgoers() (and/or incomers())
const { graph, instanceName } = this.props;
const { localGraph } = this.state;
if ((!graph && !localGraph) || !instanceName) {

View file

@ -4,6 +4,20 @@ export const DESKTOP_WIDTH_THRESHOLD = 1000;
export const DEFAULT_NODE_COLOR = "#CED9E0";
export const SELECTED_NODE_COLOR = "#48AFF0";
// From https://blueprintjs.com/docs/#core/colors.qualitative-color-schemes
export const QUALITATIVE_COLOR_SCHEME = [
"#2965CC",
"#29A634",
"#D99E0B",
"#D13913",
"#8F398F",
"#00B3A4",
"#DB2C6F",
"#9BBF30",
"#96622D",
"#7157D9"
];
export const INSTANCE_DOMAIN_PATH = "/instance/:domain";
export interface IInstanceDomainPath {
domain: string;

View file

@ -33,6 +33,7 @@ export interface ISearchResultInstance {
name: string;
description?: string;
userCount?: number;
type?: string;
}
export interface IInstanceDetails {

18
frontend/src/types.ts Normal file
View file

@ -0,0 +1,18 @@
export interface IColorSchemeType {
// The name of the coloring, e.g. "Instance type"
name: string;
// The name of the key in a cytoscape node's `data` field to color by.
// For example, use cytoscapeDataKey: "type" to color according to type.
cytoscapeDataKey: string;
// The values the color scheme is used for. E.g. ["mastodon", "pleroma", "misskey"].
values: string[];
}
export const typeColorScheme: IColorSchemeType = {
cytoscapeDataKey: "type",
name: "Instance type",
// We could also extract the values from the server response, but this would slow things down...
values: ["mastodon", "gab", "pleroma"]
};
export const colorSchemes: IColorSchemeType[] = [typeColorScheme];

View file

@ -18,3 +18,5 @@ export const getFromApi = (path: string): Promise<any> => {
export const domainMatchSelector = createMatchSelector<IAppState, IInstanceDomainPath>(INSTANCE_DOMAIN_PATH);
export const isSmallScreen = window.innerWidth < DESKTOP_WIDTH_THRESHOLD;
export const capitalize = (s: string): string => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase();