Merge branch 'dev'

Dockerfile

@@ -1,12 +1,12 @@
-FROM maven:3.8.6-eclipse-temurin-17 as builder
+FROM maven:3.9.5-eclipse-temurin-21 as builder
 
 WORKDIR /usr/src/app
 COPY . .
 ARG MAVEN_OPTS
 RUN mvn package -DskipTests=true
 RUN java LibVipsDownloader.java
 
-FROM eclipse-temurin:17-jdk
+FROM eclipse-temurin:21-jdk
 
 SHELL ["bash", "-c"]
 RUN mkdir -p /opt/smithereen

README.md

@@ -11,11 +11,12 @@ If you have any questions or feedback, there's a [Telegram chat](https://t.me/Sm
 ### Running directly on your server
 
 1. Install and configure MySQL
-2. Install maven and JDK >=17 if you don't have it already
+2. Install maven and JDK >=21 if you don't have it already
 3. Build the jar by running `mvn package -DskipTests=true` and place the one with dependencies at `/opt/smithereen/smithereen.jar`
 4. Set up the image processing native library ([libvips](https://github.com/libvips/libvips)): run `java LibVipsDownloader.java` to automatically download a prebuilt one from [here](https://github.com/lovell/sharp-libvips). If you already have libvips installed on your system, you may skip this step, but be aware that not all libvips builds include all the features Smithereen needs.
 5. Install and configure [imgproxy](https://docs.imgproxy.net/GETTING_STARTED)
 6. Fill in the config file, see a commented example [here](examples/config.properties)
+	- You can use either the local file system (default) or an S3-compatible object storage service for user-uploaded media files.
 7. Create a new MySQL database and initialize it with the [schema](schema.sql) using a command (`mysql -p smithereen < schema.sql`) or any GUI like phpMyAdmin
 8. Configure and start the daemon: assuming your distribution uses systemd, copy [the service file](examples/smithereen.service) to /etc/systemd/system, then run `systemctl daemon-reload` and `service smithereen start`
 9. Run `java -jar /opt/smithereen/smithereen.jar /etc/smithereen/config.properties init_admin` to create the first account
@@ -25,6 +26,37 @@ If you have any questions or feedback, there's a [Telegram chat](https://t.me/Sm
 
 Copy [Docker-specific config example](examples/config_docker.properties) to the project root directory as `config.properties` and edit it to set your domain. Also edit `docker-compose.yml` to add your imgproxy secrets. You can then use `docker-compose` to run Smithereen, MySQL, and imgproxy. You still need to [configure your web server to reverse proxy the port 4567](examples/nginx.conf). Create the first account by running `docker-compose exec web bash -c ./smithereen-init-admin`.
 
+### Using S3 object storage
+
+Smithereen supports S3-compatible object storage for user-uploaded media files (but not media file cache for files downloaded from other servers).
+
+To enable S3 storage, set `upload.backend=s3` in your `config.properties`. Configure other properties depending on your cloud provider:
+- `upload.s3.region`: the region to use, `us-east-1` by default. Required for AWS, but some other cloud providers accept arbitrary values here.
+- `upload.s3.endpoint`: the S3 endpoint, `s3.<region>.amazonaws.com` by default. Required if **not** using AWS.
+- `upload.s3.key_id` and `upload.s3.secret_key`: your credentials for request authentication.
+- `upload.s3.bucket`: the name of your bucket.
+- `upload.s3.override_path_style`: if `upload.s3.endpoint` is set, set this to `true` if your cloud provider requires 
+putting the bucket name into the hostname instead of in the path for API requests, like `<bucket>.<endpoint>`.
+- `upload.s3.protocol`: `https` by default, can be set to `http`.
+
+The following properties control the public URLs for clients to read the files from your S3 bucket. They're currently only used for imgproxy, but will be given out to clients directly when support for non-image (e.g. video) attachments arrives in a future Smithereen version:
+- `upload.s3.hostname`: defaults to `s3-<region>.amazonaws.com`. Needs to be set if not using AWS and `upload.s3.alias_host` is not set.
+- `upload.s3.alias_host`: can be used instead of `upload.s3.hostname` if you don't want your bucket name to be visible. Requires that you have a CDN or a reverse proxy in front of the storage provider.
+  - If this is set, the bucket name is **not** included in the generated URLs. The URLs will have the form of `<protocol>://<alias_host>/<object>`.
+  - If this is **not** set, the generated URLs will be of the form `<protocol>://<hostname>/<bucket>/<object>`.
+
+You will need to configure your bucket to allow anonymous read access to objects, but not allow directory listing. [Refer to Mastodon documentation on how to do this on different cloud providers.](https://docs.joinmastodon.org/admin/optional/object-storage/#minio)
+
+You will also need to configure imgproxy to allow it to access your S3 storage:
+```
+IMGPROXY_ALLOWED_SOURCES=local://,https://<your S3 hostname or alias host>/
+```
+[Make sure to include a trailing slash in the URL.](https://docs.imgproxy.net/configuration/options#IMGPROXY_ALLOWED_SOURCES)
+
 ## Contributing
 
 If you would like to help translate Smithereen into your language, please [do so on Crowdin](https://crowdin.com/project/smithereen).
+
+## Federating with Smithereen
+
+Smithereen supports various features not found in most other ActivityPub server software. [See the federation document](/FEDERATION.md) if you would like to implement these ActivityPub extensions in your project.

examples/config.properties

@@ -14,6 +14,9 @@ domain=YOUR_DOMAIN_HERE
 
 # Filesystem path where user-uploaded files (profile pictures, post media) are stored.
 upload.path=/opt/smithereen/media/uploads
+# Or, if you want to use S3 object storage, see readme for details
+#upload.backend=s3
+#upload.s3. ...
 
 # Media cache temporarily stores files from other servers
 media_cache.path=/opt/smithereen/media/media_cache

examples/config_docker.properties

@@ -16,6 +16,9 @@ domain=YOUR_DOMAIN_HERE
 
 # Filesystem path where user-uploaded files (profile pictures, post media) are stored.
 upload.path=/opt/smithereen/media/uploads
+# Or, if you want to use S3 object storage, see readme for details
+#upload.backend=s3
+#upload.s3. ...
 
 # Media cache temporarily stores files from other servers
 media_cache.path=/opt/smithereen/media/media_cache

examples/nginx.conf

@@ -5,6 +5,7 @@ server{
     server_name YOUR_DOMAIN_HERE;
 
     client_max_body_size 40M;
+    client_body_buffer_size 1M;
     location /i/ {
         proxy_pass http://127.0.0.1:4560;
         proxy_cache sm_images;

pom.xml

@@ -6,8 +6,8 @@
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<maven.compiler.source>17</maven.compiler.source>
-		<maven.compiler.target>17</maven.compiler.target>
+		<maven.compiler.source>21</maven.compiler.source>
+		<maven.compiler.target>21</maven.compiler.target>
 		<maven.install.skip>true</maven.install.skip>
 <!--		<skipTests>true</skipTests>-->
 	</properties>
@@ -112,7 +112,7 @@
 				<artifactId>frontend-maven-plugin</artifactId>
 				<!-- Use the latest released version:
 				https://repo1.maven.org/maven2/com/github/eirslett/frontend-maven-plugin/ -->
-				<version>1.12.0</version>
+				<version>1.14.2</version>
 				<configuration>
 					<nodeVersion>v16.13.1</nodeVersion>
 					<workingDirectory>src/main/web</workingDirectory>
@@ -190,7 +190,7 @@
 		<dependency>
 			<groupId>me.grishka.sparkjava</groupId>
 			<artifactId>spark-core</artifactId>
-			<version>2.9.4+patch.1</version>
+			<version>2.9.4+patch.2</version>
 		</dependency>
 		<dependency>
 			<groupId>io.pebbletemplates</groupId>
@@ -214,11 +214,6 @@
 			<artifactId>jsoup</artifactId>
 			<version>1.15.3</version>
 		</dependency>
-		<dependency>
-			<groupId>com.squareup.okhttp3</groupId>
-			<artifactId>okhttp</artifactId>
-			<version>3.14.9</version>
-		</dependency>
 		<dependency>
 			<groupId>org.junit.jupiter</groupId>
 			<artifactId>junit-jupiter</artifactId>

schema.sql

@@ -16,16 +16,54 @@ CREATE TABLE `accounts` (
   `password` binary(32) DEFAULT NULL,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `invited_by` int unsigned DEFAULT NULL,
-  `access_level` tinyint unsigned NOT NULL DEFAULT '1',
   `preferences` text,
   `last_active` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `ban_info` text,
   `activation_info` json DEFAULT NULL,
+  `role` int unsigned DEFAULT NULL,
+  `promoted_by` int unsigned DEFAULT NULL,
+  `email_domain` varchar(150) NOT NULL DEFAULT '',
+  `last_ip` binary(16) DEFAULT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `email` (`email`),
   KEY `user_id` (`user_id`),
   KEY `invited_by` (`invited_by`),
-  CONSTRAINT `accounts_ibfk_1` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE
+  KEY `role` (`role`),
+  KEY `promoted_by` (`promoted_by`),
+  KEY `email_domain` (`email_domain`),
+  KEY `last_ip` (`last_ip`),
+  CONSTRAINT `accounts_ibfk_1` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE,
+  CONSTRAINT `accounts_ibfk_2` FOREIGN KEY (`role`) REFERENCES `user_roles` (`id`) ON DELETE SET NULL,
+  CONSTRAINT `accounts_ibfk_3` FOREIGN KEY (`promoted_by`) REFERENCES `accounts` (`id`) ON DELETE SET NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `audit_log`
+--
+
+CREATE TABLE `audit_log` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `admin_id` int unsigned NOT NULL,
+  `action` int unsigned NOT NULL,
+  `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `owner_id` int DEFAULT NULL,
+  `object_id` bigint DEFAULT NULL,
+  `object_type` int unsigned DEFAULT NULL,
+  `extra` json DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `owner_id` (`owner_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `blocks_email_domain`
+--
+
+CREATE TABLE `blocks_email_domain` (
+  `domain` varchar(100) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+  `action` tinyint unsigned NOT NULL,
+  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `note` text NOT NULL,
+  `creator_id` int unsigned NOT NULL,
+  PRIMARY KEY (`domain`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 --
@@ -53,6 +91,22 @@ CREATE TABLE `blocks_group_user` (
   CONSTRAINT `blocks_group_user_ibfk_2` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+--
+-- Table structure for table `blocks_ip`
+--
+
+CREATE TABLE `blocks_ip` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `address` binary(16) NOT NULL,
+  `prefix_length` tinyint unsigned NOT NULL,
+  `action` tinyint unsigned NOT NULL,
+  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `expires_at` timestamp NOT NULL,
+  `note` text NOT NULL,
+  `creator_id` int unsigned NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Table structure for table `blocks_user_domain`
 --
@@ -329,6 +383,42 @@ CREATE TABLE `media_cache` (
   KEY `last_access` (`last_access`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+--
+-- Table structure for table `media_file_refs`
+--
+
+CREATE TABLE `media_file_refs` (
+  `file_id` bigint unsigned NOT NULL,
+  `object_id` bigint NOT NULL,
+  `object_type` tinyint unsigned NOT NULL,
+  `owner_user_id` int unsigned DEFAULT NULL,
+  `owner_group_id` int unsigned DEFAULT NULL,
+  PRIMARY KEY (`object_id`,`object_type`,`file_id`),
+  KEY `file_id` (`file_id`),
+  KEY `owner_user_id` (`owner_user_id`),
+  KEY `owner_group_id` (`owner_group_id`),
+  CONSTRAINT `media_file_refs_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `media_files` (`id`),
+  CONSTRAINT `media_file_refs_ibfk_2` FOREIGN KEY (`owner_user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE,
+  CONSTRAINT `media_file_refs_ibfk_3` FOREIGN KEY (`owner_group_id`) REFERENCES `groups` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `media_files`
+--
+
+CREATE TABLE `media_files` (
+  `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+  `random_id` binary(18) NOT NULL,
+  `size` bigint unsigned NOT NULL,
+  `type` tinyint unsigned NOT NULL,
+  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `metadata` json NOT NULL,
+  `ref_count` int unsigned NOT NULL DEFAULT '0',
+  `original_owner_id` int NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `ref_count` (`ref_count`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Table structure for table `newsfeed`
 --
@@ -447,6 +537,23 @@ CREATE TABLE `qsearch_index` (
   CONSTRAINT `qsearch_index_ibfk_2` FOREIGN KEY (`group_id`) REFERENCES `groups` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=ascii;
 
+--
+-- Table structure for table `report_actions`
+--
+
+CREATE TABLE `report_actions` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `report_id` int unsigned NOT NULL,
+  `user_id` int unsigned NOT NULL,
+  `action_type` tinyint unsigned NOT NULL,
+  `text` text,
+  `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `extra` json DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `report_id` (`report_id`),
+  CONSTRAINT `report_actions_ibfk_1` FOREIGN KEY (`report_id`) REFERENCES `reports` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Table structure for table `reports`
 --
@@ -455,17 +562,18 @@ CREATE TABLE `reports` (
   `id` int unsigned NOT NULL AUTO_INCREMENT,
   `reporter_id` int unsigned DEFAULT NULL,
   `target_type` tinyint unsigned NOT NULL,
-  `content_type` tinyint unsigned DEFAULT NULL,
-  `target_id` int unsigned NOT NULL,
-  `content_id` bigint unsigned DEFAULT NULL,
+  `target_id` int NOT NULL,
   `comment` text NOT NULL,
   `moderator_id` int unsigned DEFAULT NULL,
   `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `action_time` timestamp NULL DEFAULT NULL,
   `server_domain` varchar(100) DEFAULT NULL,
+  `content` json DEFAULT NULL,
+  `state` tinyint unsigned NOT NULL DEFAULT '0',
   PRIMARY KEY (`id`),
   KEY `reporter_id` (`reporter_id`),
-  KEY `moderator_id` (`moderator_id`)
+  KEY `moderator_id` (`moderator_id`),
+  KEY `state` (`state`),
+  KEY `target_id` (`target_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 --
@@ -498,7 +606,8 @@ CREATE TABLE `sessions` (
   `id` binary(64) NOT NULL,
   `account_id` int unsigned NOT NULL,
   `last_active` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `last_ip` varbinary(16) DEFAULT NULL,
+  `ip` binary(16) NOT NULL,
+  `user_agent` bigint NOT NULL,
   PRIMARY KEY (`id`),
   KEY `account_id` (`account_id`),
   CONSTRAINT `sessions_ibfk_1` FOREIGN KEY (`account_id`) REFERENCES `accounts` (`id`) ON DELETE CASCADE
@@ -550,6 +659,42 @@ CREATE TABLE `stats_daily` (
   PRIMARY KEY (`day`,`type`,`object_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+--
+-- Table structure for table `user_agents`
+--
+
+CREATE TABLE `user_agents` (
+  `hash` bigint NOT NULL,
+  `user_agent` text NOT NULL,
+  PRIMARY KEY (`hash`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `user_roles`
+--
+
+CREATE TABLE `user_roles` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `permissions` varbinary(255) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `user_staff_notes`
+--
+
+CREATE TABLE `user_staff_notes` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `target_id` int unsigned NOT NULL,
+  `author_id` int unsigned NOT NULL,
+  `text` text NOT NULL,
+  `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `target_id` (`target_id`),
+  CONSTRAINT `user_staff_notes_ibfk_1` FOREIGN KEY (`target_id`) REFERENCES `users` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Table structure for table `users`
 --
@@ -578,9 +723,12 @@ CREATE TABLE `users` (
   `flags` bigint unsigned NOT NULL,
   `endpoints` json DEFAULT NULL,
   `privacy` json DEFAULT NULL,
+  `ban_status` tinyint unsigned NOT NULL DEFAULT '0',
+  `ban_info` json DEFAULT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `username` (`username`,`domain`),
-  UNIQUE KEY `ap_id` (`ap_id`)
+  UNIQUE KEY `ap_id` (`ap_id`),
+  KEY `ban_status` (`ban_status`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 --
@@ -624,4 +772,4 @@ CREATE TABLE `wall_posts` (
 
 /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
 
--- Dump completed on 2023-10-26 17:12:16
+-- Dump completed on 2024-03-23  8:50:17