Merge branch 'dev'

FEDERATION.md

@@ -53,8 +53,6 @@ Friend requests are sent as `Offer{Follow}` activities, where the inner `Follow`
 Accepting a friend request is done by following the sender back, that is, simply sending a `Follow`.
 
 Rejecting a friend request is done by sending a `Reject{Offer{Follow}}` activity.
-### Additional profile fields
-There are separate fields for first and last names, birth date, and gender, all based on schema.org. Those are `firstName`, `lastName`, `birthDate` and `gender`. `firstName` and `lastName` are respectively aliases to `givenName` and `familyName`. Birth date is in the format `YYYY-MM-DD`. Gender can either be `sc:Male` or `sc:Female`.
 ### Non-square profile pictures
 Smithereen uses non-square profile pictures on the profile page. In order to retain compatibility with everything else, `icon` in the actor still points to a square picture. It's extended with the `image` field that contains the full rectangular one, and `sm:cropRegion` with the coordinates of the square version within the rectangular one. The coordinates are in the order `[x1, y1, x2, y2]`, where (x1, y1) are the top-left corner of the square, and (x2, y2) are the bottom-right. The top-left corner of the rectangle is (0, 0), and the bottom-right one is (1, 1).
 
@@ -262,4 +260,94 @@ If a server still sends an activity that is not allowed by the user's privacy se
 If a user sends a message to someone who can't send **them** messages, the recipient is temporarily allowed to send them messages. The exception lasts for a week (168 hours) or 10 messages, whichever comes first. Every outgoing message resets the time and the message count.
 
 ### Read receipts for direct messages
-When a recipient views a direct message (a `Note` that is only addressed to actors and no collections) for the first time, Smithereen will send a `Read{Note}` activity to its sender.
+When a recipient views a direct message (a `Note` that is only addressed to actors and no collections) for the first time, Smithereen will send a `Read{Note}` activity to its sender.
+
+### User profile fields
+
+Additional profile fields are mostly just custom keys in `Person` actors. Just like many other things in this project, they are mostly copied from VKontakte. Unless otherwise noted, the values are free-form plain (**non-HTML**) strings.
+
+##### Structured name
+
+- `firstName`, alias to `sc:givenName`
+- `lastName`, alias to `sc:familyName`
+- `middleName`, alias to `sc:additionalName`
+- `sm:maidenName`
+
+##### Gender
+
+`sc:gender`, not displayed anywhere but is used to select pronouns and inflect names, possible values:
+- `sc:Male`, he/him
+- `sc:Female`, she/her
+- `sc:Other`, they/them
+
+##### "Main" unlabeled section below the name
+
+- `vcard:bday` — birth date, of the form YYYY-MM-DD
+- `sm:hometown`
+- `sm:relationshipStatus`, possible values:
+  - `sm:Single`
+  - `sm:InRelationship`
+  - `sm:Engaged`
+  - `sm:Married`
+  - `sm:InLove`
+  - `sm:Complicated`
+  - `sm:ActivelySearching`
+- `sm:relationshipPartner` — user's partner, must be an ID of another `Person` actor. Only valid when `sm:relationshipStatus` is present and is other than `sm:Single` or `sm:ActivelySearching`. For the partner to show up in the UI, they also need to set this user as their partner, *unless* the relationship status is `sm:InLove`.
+
+If the actor has any unrecognized `PropertyValue` fields, they are also displayed in this section.
+
+##### "Contacts" section
+
+- `vcard:Address` — city/location. Pleroma and Misskey had this field before Smithereen.
+
+Other fields (website, Matrix, ...) are Mastodon-compatible `PropertyValue`s with hardcoded English names and HTML links to corresponding services.
+
+##### "Personal"/"philosophy" section
+
+- `sm:politicalViews`, possible values:
+  - `sm:Apathetic`
+  - `sm:Communist`
+  - `sm:Socialist`
+  - `sm:Moderate`
+  - `sm:Liberal`
+  - `sm:Conservative`
+  - `sm:Monarchist`
+  - `sm:Ultraconservative`
+  - `sm:Libertarian`
+- `sm:religion`
+- `sm:personalPriority`, possible values:
+  - `sm:FamilyAndChildren`
+  - `sm:CareerAndMoney`
+  - `sm:EntertainmentAndLeisure`
+  - `sm:ScienceAndResearch`
+  - `sm:ImprovingTheWorld`
+  - `sm:PersonalDevelopment`
+  - `sm:BeautyAndArt`
+  - `sm:FameAndInfluence`
+- `sm:peoplePriority`, possible values:
+  - `sm:IntellectAndCreativity`
+  - `sm:KindnessAndHonesty`
+  - `sm:HealthAndBeauty`
+  - `sm:WealthAndPower`
+  - `sm:CourageAndPersistence`
+  - `sm:HumorAndLoveForLife`
+- `sm:smokingViews`, possible values:
+  - `sm:VeryNegative`
+  - `sm:Negative`
+  - `sm:Tolerant`
+  - `sm:Neutral`
+  - `sm:Positive`
+- `sm:alcoholViews`, the values are the same as for `sm:smokingViews`
+- `sm:inspiredBy`
+
+##### "Interests" section
+
+- `sm:activities`
+- `sm:interests`
+- `sm:favoriteMusic`
+- `sm:favoriteMovies`
+- `sm:favoriteTvShows`
+- `sm:favoriteBooks`
+- `sm:favoriteGames`
+- `sm:favoriteQuotes`
+- `summary` — the standard ActivityPub "bio" field, this is an HTML string

examples/config.properties

@@ -3,6 +3,8 @@ db.host=localhost
 db.name=smithereen
 db.user=root
 db.password=mysql
+# You can limit the number of concurrent database connections, default is 100
+#db.max_connections=50
 
 # The domain for your instance. Used for local object URIs in ActivityPub. If running on localhost, must include the port.
 #domain=localhost:4567

pom.xml

@@ -86,7 +86,7 @@
 							<executable>scp</executable>
 							<arguments>
 								<argument>${basedir}/target/smithereen-jar-with-dependencies.jar</argument>
-								<argument>root@${deployHost}:/opt/smithereen/smithereen.jar</argument>
+								<argument>root@${deployHost}:/opt/smithereen/smithereen_new.jar</argument>
 							</arguments>
 						</configuration>
 					</execution>
@@ -100,7 +100,7 @@
 							<executable>ssh</executable>
 							<arguments>
 								<argument>root@${deployHost}</argument>
-								<argument>service smithereen restart</argument>
+								<argument>service smithereen stop; mv /opt/smithereen/smithereen_new.jar /opt/smithereen/smithereen.jar; service smithereen start</argument>
 							</arguments>
 						</configuration>
 					</execution>
@@ -245,6 +245,21 @@
 			<artifactId>jna</artifactId>
 			<version>5.11.0</version>
 		</dependency>
+		<dependency>
+			<groupId>org.commonmark</groupId>
+			<artifactId>commonmark</artifactId>
+			<version>0.22.0</version>
+		</dependency>
+		<dependency>
+			<groupId>org.commonmark</groupId>
+			<artifactId>commonmark-ext-gfm-strikethrough</artifactId>
+			<version>0.22.0</version>
+		</dependency>
+		<dependency>
+			<groupId>org.commonmark</groupId>
+			<artifactId>commonmark-ext-ins</artifactId>
+			<version>0.22.0</version>
+		</dependency>
 	</dependencies>
 
 

schema.sql

@@ -132,13 +132,41 @@ CREATE TABLE `blocks_user_user` (
   CONSTRAINT `blocks_user_user_ibfk_2` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+--
+-- Table structure for table `bookmarks_group`
+--
+
+CREATE TABLE `bookmarks_group` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `owner_id` int unsigned NOT NULL,
+  `group_id` int unsigned NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `owner_id_2` (`owner_id`,`group_id`),
+  KEY `group_id` (`group_id`),
+  CONSTRAINT `bookmarks_group_ibfk_1` FOREIGN KEY (`owner_id`) REFERENCES `users` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `bookmarks_user`
+--
+
+CREATE TABLE `bookmarks_user` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `owner_id` int unsigned NOT NULL,
+  `user_id` int unsigned NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `owner_id` (`owner_id`,`user_id`),
+  KEY `user_id` (`user_id`),
+  CONSTRAINT `bookmarks_user_ibfk_1` FOREIGN KEY (`owner_id`) REFERENCES `users` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Table structure for table `config`
 --
 
 CREATE TABLE `config` (
   `key` varchar(255) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL DEFAULT '',
-  `value` text NOT NULL,
+  `value` mediumtext NOT NULL,
   PRIMARY KEY (`key`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
@@ -267,7 +295,7 @@ CREATE TABLE `group_memberships` (
 CREATE TABLE `groups` (
   `id` int unsigned NOT NULL AUTO_INCREMENT,
   `name` varchar(200) NOT NULL DEFAULT '',
-  `username` varchar(50) NOT NULL DEFAULT '',
+  `username` varchar(64) NOT NULL,
   `domain` varchar(100) NOT NULL DEFAULT '',
   `ap_id` varchar(300) CHARACTER SET ascii COLLATE ascii_general_ci DEFAULT NULL,
   `ap_url` varchar(300) DEFAULT NULL,
@@ -706,7 +734,7 @@ CREATE TABLE `users` (
   `middle_name` varchar(100) DEFAULT NULL,
   `maiden_name` varchar(100) DEFAULT NULL,
   `bdate` date DEFAULT NULL,
-  `username` varchar(50) NOT NULL DEFAULT '',
+  `username` varchar(64) NOT NULL,
   `domain` varchar(100) NOT NULL DEFAULT '',
   `public_key` blob NOT NULL,
   `private_key` blob,
@@ -757,6 +785,7 @@ CREATE TABLE `wall_posts` (
   `source` text,
   `source_format` tinyint unsigned DEFAULT NULL,
   `privacy` tinyint unsigned NOT NULL DEFAULT '0',
+  `flags` bigint unsigned NOT NULL DEFAULT '0',
   PRIMARY KEY (`id`),
   UNIQUE KEY `ap_id` (`ap_id`),
   KEY `owner_user_id` (`owner_user_id`),
@@ -766,10 +795,9 @@ CREATE TABLE `wall_posts` (
   KEY `owner_group_id` (`owner_group_id`),
   KEY `poll_id` (`poll_id`),
   CONSTRAINT `wall_posts_ibfk_1` FOREIGN KEY (`owner_user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE,
-  CONSTRAINT `wall_posts_ibfk_2` FOREIGN KEY (`repost_of`) REFERENCES `wall_posts` (`id`) ON DELETE SET NULL,
   CONSTRAINT `wall_posts_ibfk_4` FOREIGN KEY (`owner_group_id`) REFERENCES `groups` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
 
--- Dump completed on 2024-03-23  8:50:17
+-- Dump completed on 2024-06-06 11:51:39

src/main/java/smithereen/ApplicationContext.java

@@ -1,6 +1,7 @@
 package smithereen;
 
 import smithereen.activitypub.ActivityPubWorker;
+import smithereen.controllers.BookmarksController;
 import smithereen.controllers.FriendsController;
 import smithereen.controllers.GroupsController;
 import smithereen.controllers.MailController;
@@ -9,6 +10,7 @@
 import smithereen.controllers.NotificationsController;
 import smithereen.controllers.ObjectLinkResolver;
 import smithereen.controllers.PrivacyController;
+import smithereen.controllers.SearchController;
 import smithereen.controllers.StatsController;
 import smithereen.controllers.UserInteractionsController;
 import smithereen.controllers.UsersController;
@@ -28,6 +30,8 @@ public class ApplicationContext{
 	private final ModerationController moderationController;
 	private final StatsController statsController;
 	private final MailController mailController;
+	private final SearchController searchController;
+	private final BookmarksController bookmarksController;
 
 	public ApplicationContext(){
 		wallController=new WallController(this);
@@ -43,6 +47,8 @@ public ApplicationContext(){
 		moderationController=new ModerationController(this);
 		statsController=new StatsController(this);
 		mailController=new MailController(this);
+		searchController=new SearchController(this);
+		bookmarksController=new BookmarksController(this);
 	}
 
 	public WallController getWallController(){
@@ -96,4 +102,12 @@ public StatsController getStatsController(){
 	public MailController getMailController(){
 		return mailController;
 	}
+
+	public SearchController getSearchController(){
+		return searchController;
+	}
+
+	public BookmarksController getBookmarksController(){
+		return bookmarksController;
+	}
 }

src/main/java/smithereen/Config.java

@@ -26,6 +26,7 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
@@ -40,6 +41,7 @@
 import smithereen.storage.sql.SQLQueryBuilder;
 import smithereen.storage.sql.DatabaseConnection;
 import smithereen.storage.sql.DatabaseConnectionManager;
+import smithereen.util.PublicSuffixList;
 import smithereen.util.TopLevelDomainList;
 import spark.utils.StringUtils;
 
@@ -48,6 +50,7 @@ public class Config{
 	public static String dbUser;
 	public static String dbPassword;
 	public static String dbName;
+	public static int dbMaxConnections;
 
 	public static String domain;
 
@@ -111,6 +114,7 @@ public static void load(String filePath) throws IOException{
 		dbUser=props.getProperty("db.user");
 		dbPassword=props.getProperty("db.password");
 		dbName=props.getProperty("db.name");
+		dbMaxConnections=Utils.parseIntOrDefault(props.getProperty("db.max_connections"), 100);
 
 		domain=props.getProperty("domain");
 
@@ -238,6 +242,11 @@ public static void loadFromDatabase() throws SQLException{
 			if(TopLevelDomainList.lastUpdatedTime>0){
 				TopLevelDomainList.update(dbValues.get("TLDList_Data"));
 			}
+
+			PublicSuffixList.lastUpdatedTime=Long.parseLong(dbValues.getOrDefault("PSList_LastUpdated", "0"));
+			if(PublicSuffixList.lastUpdatedTime>0){
+				PublicSuffixList.update(Arrays.asList(dbValues.get("PSList_Data").split("\n")));
+			}
 		}
 	}
 

src/main/java/smithereen/Mailer.java

@@ -28,6 +28,7 @@
 import javax.mail.internet.MimeMultipart;
 
 import smithereen.model.Account;
+import smithereen.text.TextProcessor;
 import smithereen.util.UriBuilder;
 import smithereen.lang.Lang;
 import smithereen.model.UserBanInfo;
@@ -156,11 +157,11 @@ public void sendEmailChangeDoneToPreviousAddress(Request req, Account self){
 	public void sendSignupInvitation(Request req, Account self, String email, String code, String firstName, boolean isRequest){
 		Lang l=Utils.lang(req);
 		String link=UriBuilder.local().path("account", "register").queryParam("invite", code).build().toString();
-		String plaintext=Utils.stripHTML(l.get(isRequest ? "email_invite_body_start_approved" : "email_invite_body_start", Map.of(
+		String plaintext=TextProcessor.stripHTML(l.get(isRequest ? "email_invite_body_start_approved" : "email_invite_body_start", Map.of(
 				"name", firstName,
 				"inviterName", self.user.getFullName(),
 				"serverName", Config.serverDisplayName
-		)));
+		)), true);
 		plaintext+="\n\n"+l.get("email_invite_body_end_plain")+"\n\n"+link;
 		send(email, l.get(isRequest ? "email_invite_subject_approved" : "email_invite_subject", Map.of("serverName", Config.serverDisplayName)), plaintext, "signup_invitation", Map.of(
 				"name", firstName,
@@ -192,9 +193,9 @@ public void sendAccountBanNotification(Account self, UserBanStatus banStatus, Us
 			default -> throw new IllegalArgumentException("Unexpected value: " + banStatus);
 		};
 		String htmlText=text;
-		text=Utils.stripHTML(text);
+		text=TextProcessor.stripHTML(text, true);
 		if(StringUtils.isNotEmpty(banInfo.message())){
-			htmlText+="<br/><br/>"+l.get("message_from_staff")+": "+Utils.stripHTML(banInfo.message());
+			htmlText+="<br/><br/>"+l.get("message_from_staff")+": "+TextProcessor.stripHTML(banInfo.message(), true);
 			text+="\n\n"+l.get("message_from_staff")+": "+banInfo.message();
 		}
 		send(self.email, subject, text, "generic", Map.of("text", htmlText), self.prefs.locale);
@@ -203,7 +204,7 @@ public void sendAccountBanNotification(Account self, UserBanStatus banStatus, Us
 	public void sendActionConfirmationCode(Request req, Account self, String action, String code){
 		LOG.trace("Sending code {} for action {}", code, action);
 		Lang l=Utils.lang(req);
-		String plaintext=Utils.stripHTML(l.get("email_confirmation_code", Map.of("name", self.user.firstName, "action", action)))+"\n\n"+code+"\n\n"+Utils.stripHTML(l.get("email_confirmation_code_info"));
+		String plaintext=TextProcessor.stripHTML(l.get("email_confirmation_code", Map.of("name", self.user.firstName, "action", action)), true)+"\n\n"+code+"\n\n"+TextProcessor.stripHTML(l.get("email_confirmation_code_info"), true);
 		String subject=l.get("email_confirmation_code_subject", Map.of("domain", Config.domain));
 		send(self.email, subject, plaintext, "confirmation_code", Map.of("name", self.user.firstName, "action", action, "code", code), l.getLocale());
 	}