inex
/
index.community
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move to dokku's DATABASE_URL format

This commit is contained in:
Tao Bror Bojlén 2019-07-14 12:36:05 +01:00
parent aea1382131
commit 4fb2d71d89
No known key found for this signature in database
GPG Key ID: C6EC7AAB905F9E6F
11 changed files with 91 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.dokku-monorepo Normal file
View File

@ -0,0 +1,2 @@
backend=backend
gephi=gephi

17
backend/config/releases.exs
View File

@ -7,14 +7,17 @@ import Config
# to something meaningful, Phoenix uses this information
# when generating URLs.
config :backend, Backend.Repo,
username: System.get_env("POSTGRES_USER"),
password: System.get_env("POSTGRES_PASSWORD"),
database: System.get_env("POSTGRES_DB"),
hostname: System.get_env("POSTGRES_HOSTNAME"),
pool_size: 10,
show_sensitive_data_on_connection_error: true
# username: System.get_env("POSTGRES_USER"),
# password: System.get_env("POSTGRES_PASSWORD"),
# database: System.get_env("POSTGRES_DB"),
# hostname: System.get_env("POSTGRES_HOSTNAME"),
url: System.get_env("ecto://" <> "DATABASE_URL"),
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
ssl: true
port = String.to_integer(System.get_env("BACKEND_PORT") || "4000")
# show_sensitive_data_on_connection_error: true
port = String.to_integer(System.get_env("PORT") || "4000")
config :backend, BackendWeb.Endpoint,
http: [:inet6, port: port],

13
config/Caddyfile
View File

@ -1,13 +0,0 @@
backend.fediverse.space {
tls tao@btao.org
gzip
cors
root /srv
proxy / django:8000 {
transparent
except /static
}
}

29
docker-compose.production.yml
View File

@ -9,34 +9,15 @@ services:
build: ./backend
networks:
- database_network
- server_network
depends_on:
- db
ports:
- "4000:4000"
- "${PORT}:${PORT}"
environment:
- POSTGRES_USER
- POSTGRES_PASSWORD
- POSTGRES_DB
- POSTGRES_HOSTNAME=db
- DATABASE_URL
- SECRET_KEY_BASE
- PORT
caddy:
restart: always
image: abiosoft/caddy:0.11.4-no-stats
ports:
- "80:80"
- "443:443"
volumes:
- ./config/Caddyfile:/etc/Caddyfile
- caddycerts:/etc/caddycerts
networks:
- server_network
depends_on:
- phoenix
environment:
- ACME_AGREE
- CADDYPATH=/etc/caddycerts
- BACKEND_HOSTNAME
gephi:
networks:
- database_network
@ -45,7 +26,3 @@ services:
networks:
database_network:
driver: bridge
server_network:
driver: bridge
volumes:
caddycerts:

8
docker-compose.yml
View File

@ -3,9 +3,7 @@ services:
db:
image: postgres
environment:
- POSTGRES_USER
- POSTGRES_PASSWORD
- POSTGRES_DB
- DATABASE_URL
ports:
- "5432:5432"
volumes:
@ -14,9 +12,7 @@ services:
# as everything else, but it should be run regularly with a cron job or similar.
gephi:
environment:
- POSTGRES_USER
- POSTGRES_PASSWORD
- POSTGRES_DB
- DATABASE_URL
build: ./gephi
volumes:
- gradle-cache:/code/.gradle

9
example.env
View File

@ -1,7 +1,4 @@
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
POSTGRES_DB=backend_dev
POSTGRES_HOSTNAME=db
DATABASE_URL="postgres://postgres:postgres@localhost:5432/backend_dev"
PORT=4000
BACKEND_HOSTNAME=localhost
SECRET_KEY_BASE=jLqbBjtQTyZj+1yLwDV8xgZYvZKIBx1MBWbcC2a0mZqB5ivYKQ7GOqNR91g6YnR8
ACME_AGREE=true
SECRET_KEY_BASE=jLqbBjtQTyZj+1yLwDV8xgZYvZKIBx1MBWbcC2a0mZqB5ivYKQ7GOqNR91g6YnR8

12
gephi/.classpath Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="bin/main" path="src/main/java">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8/"/>
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
<classpathentry kind="output" path="bin/default"/>
</classpath>

23
gephi/.project Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>gephi</name>
<comment>Project gephi created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
</projectDescription>

2
gephi/.settings/org.eclipse.buildship.core.prefs Normal file
View File

@ -0,0 +1,2 @@
connection.project.dir=
eclipse.preferences.version=1

BIN
gephi/bin/main/space/fediverse/graph/GraphBuilder.class Normal file
View File

Binary file not shown.

75
gephi/src/main/java/space/fediverse/graph/GraphBuilder.java
View File

@ -21,38 +21,23 @@ import org.openide.util.Lookup;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
public class GraphBuilder {
private static final String nodeQuery = String.join(""
, "WITH successful_crawls AS ("
, " SELECT"
, " c.instance_domain AS instance_domain,"
, " COUNT(c.id) AS crawl_count"
, " FROM crawls c WHERE c.error IS NULL"
, " GROUP BY c.instance_domain)"
, " SELECT"
, " i.domain AS id,"
, " i.domain AS label"
, " FROM instances i"
, " INNER JOIN successful_crawls c ON i.domain = c.instance_domain"
, " WHERE c.crawl_count > 0 AND i.user_count IS NOT NULL"
);
private static final String nodeQuery = String.join("", "WITH successful_crawls AS (", " SELECT",
" c.instance_domain AS instance_domain,", " COUNT(c.id) AS crawl_count",
" FROM crawls c WHERE c.error IS NULL", " GROUP BY c.instance_domain)", " SELECT", " i.domain AS id,",
" i.domain AS label", " FROM instances i",
" INNER JOIN successful_crawls c ON i.domain = c.instance_domain",
" WHERE c.crawl_count > 0 AND i.user_count IS NOT NULL");
private static final String edgeQuery = String.join(""
, "SELECT"
, " e.source_domain AS source,"
, " e.target_domain AS target,"
, " e.weight AS weight"
, " FROM edges e"
);
private static final String edgeQuery = String.join("", "SELECT", " e.source_domain AS source,",
" e.target_domain AS target,", " e.weight AS weight", " FROM edges e");
public static void main(String[] args) {
@ -67,31 +52,39 @@ public class GraphBuilder {
// AttributeModel?
// Get config variables
String postgresDb = System.getenv("POSTGRES_DB");
String postgresUser = System.getenv("POSTGRES_USER");
String postgresPassword = System.getenv("POSTGRES_PASSWORD");
if (postgresDb == null || postgresUser == null || postgresPassword == null) {
throw new RuntimeException(String.format("Incomplete config, canceling. DB: %s, user: %s, pass: %s",
postgresDb, postgresUser, postgresPassword));
// DATABASE_URL has the format postgres://user:password@host:port/database
String[] databaseParams = System.getenv("DATABASE_URL").replace("postgres://", "").split(":|@|/");
String postgresUser = databaseParams[0];
String postgresPassword = databaseParams[1];
String postgresHost = databaseParams[2];
Integer postgresPort = Integer.parseInt(databaseParams[3]);
String postgresDb = databaseParams[4];
if (postgresUser == null || postgresPassword == null || postgresHost == null || postgresPort == null
|| postgresDb == null) {
throw new RuntimeException(
String.format("Incomplete config, canceling. DB: %s, user: %s, host: %s, port: %s", postgresDb,
postgresUser, postgresHost, postgresPort));
}
// Import from database
EdgeListDatabaseImpl db = new EdgeListDatabaseImpl();
db.setSQLDriver(new PostgreSQLDriver());
db.setHost("db");
db.setPort(5432);
db.setDBName(postgresDb);
db.setUsername(postgresUser);
db.setPasswd(postgresPassword);
db.setHost(postgresHost);
db.setPort(postgresPort);
db.setDBName(postgresDb);
db.setNodeQuery(nodeQuery);
db.setEdgeQuery(edgeQuery);
ImporterEdgeList edgeListImporter = new ImporterEdgeList();
Container container = importController.importDatabase(db, edgeListImporter);
// If a node is in the edge list, but not node list, we don't want to create it automatically
// If a node is in the edge list, but not node list, we don't want to create it
// automatically
container.getLoader().setAllowAutoNode(false);
container.getLoader().setAllowSelfLoop(false);
container.getLoader().setEdgeDefault(EdgeDirectionDefault.UNDIRECTED); // This is an undirected graph
container.getLoader().setEdgeDefault(EdgeDirectionDefault.UNDIRECTED); // This is an undirected graph
// Add imported data to graph
importController.process(container, new DefaultProcessor(), workspace);
@ -99,7 +92,8 @@ public class GraphBuilder {
// Layout
AutoLayout autoLayout = new AutoLayout(1, TimeUnit.MINUTES);
autoLayout.setGraphModel(graphModel);
// YifanHuLayout firstLayout = new YifanHuLayout(null, new StepDisplacement(1f));
// YifanHuLayout firstLayout = new YifanHuLayout(null, new
// StepDisplacement(1f));
ForceAtlas2 forceAtlas2Layout = new ForceAtlas2(null);
forceAtlas2Layout.setLinLogMode(true);
autoLayout.addLayout(forceAtlas2Layout, 1f);
@ -123,14 +117,13 @@ public class GraphBuilder {
}
// Update
UndirectedGraph graph = graphModel.getUndirectedGraph();
for (Node node: graph.getNodes()) {
for (Node node : graph.getNodes()) {
String id = node.getId().toString();
float x = node.x();
float y = node.y();
try {
PreparedStatement statement = conn.prepareStatement(
"UPDATE instances SET x=?, y=? WHERE domain=?");
PreparedStatement statement = conn.prepareStatement("UPDATE instances SET x=?, y=? WHERE domain=?");
statement.setFloat(1, x);
statement.setFloat(2, y);
statement.setString(3, id);
@ -146,7 +139,6 @@ public class GraphBuilder {
// Closing failed; ah well
}
// Also export to gexf
ExportController exportController = Lookup.getDefault().lookup(ExportController.class);
try {
@ -155,7 +147,8 @@ public class GraphBuilder {
throw new RuntimeException(e);
}
// Gephi doesn't seem to provide a good way to close its postgres connection, so we have to force close the
// Gephi doesn't seem to provide a good way to close its postgres connection, so
// we have to force close the
// program. This'll leave a hanging connection for some period ¯\_(ツ)_/¯
System.exit(0);
}