Merge pull request #70 from cqse/ts/41282_xcresult_missing_coverage_r…

…eworked_internals

TS-41282 XCResult coverage conversion extracts coverage from only one test plan in case there are multiple

CHANGELOG.md

@@ -7,6 +7,9 @@ We use [semantic versioning](http://semver.org/):
 # Next Release
 - ...
 
+# 2.9.5
+- [fix] Merged Xcode result bundles were not processed properly
+
 # 2.9.4
 - [fix] Uploading XCode coverage files was not possible when using Xcode version 16
 

src/main/java/com/teamscale/upload/TeamscaleUpload.java

@@ -2,7 +2,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -12,10 +11,8 @@
 import com.teamscale.upload.resolve.FilePatternResolutionException;
 import com.teamscale.upload.resolve.ReportPatternUtils;
 import com.teamscale.upload.utils.LogUtils;
-import com.teamscale.upload.xcode.ConvertedReport;
-import com.teamscale.upload.xcode.XCResultConverter;
-import com.teamscale.upload.xcode.XCResultConverter.ConversionException;
-import com.teamscale.upload.xcode.XCodeVersion;
+import com.teamscale.upload.xcode.ConversionException;
+import com.teamscale.upload.xcode.XcodeReportConverter;
 
 /**
  * Main class of the teamscale-upload project.
@@ -56,10 +53,10 @@ private static Map<String, Set<File>> resolveAndConvertFiles(CommandLine command
 
 	/**
 	 * Returns whether the given set of file formats contains the
-	 * {@linkplain XCResultConverter#XCODE_REPORT_FORMAT XCode report format}.
+	 * {@linkplain XcodeReportConverter#XCODE_REPORT_FORMAT XCode report format}.
 	 */
 	private static boolean containsAnyXCodeReports(Set<String> fileFormats) {
-		return fileFormats.contains(XCResultConverter.XCODE_REPORT_FORMAT);
+		return fileFormats.contains(XcodeReportConverter.XCODE_REPORT_FORMAT);
 	}
 
 	/**
@@ -68,18 +65,12 @@ private static boolean containsAnyXCodeReports(Set<String> fileFormats) {
 	 */
 	private static void convertXCodeReports(Map<String, Set<File>> filesByFormat) {
 		try {
-			Set<File> xcresultBundles = filesByFormat.remove(XCResultConverter.XCODE_REPORT_FORMAT);
-			XCodeVersion xcodeVersion = XCodeVersion.determine();
-			List<ConvertedReport> convertedReports = new ArrayList<>();
-			for (File xcodeReport : xcresultBundles) {
-				convertedReports.addAll(XCResultConverter.convertReport(xcodeVersion, xcodeReport));
-			}
+			Set<File> xcresultBundles = filesByFormat.remove(XcodeReportConverter.XCODE_REPORT_FORMAT);
+			List<File> convertedReports = XcodeReportConverter.convert(xcresultBundles);
 
 			// Add the converted reports back to filesByFormat
-			for (ConvertedReport convertedReport : convertedReports) {
-				filesByFormat.computeIfAbsent(convertedReport.reportFormat, format -> new HashSet<>())
-						.add(convertedReport.report);
-			}
+			filesByFormat.computeIfAbsent(XcodeReportConverter.XCODE_REPORT_FORMAT, format -> new HashSet<>())
+					.addAll(convertedReports);
 		} catch (ConversionException e) {
 			LogUtils.failWithoutStackTrace(e.getMessage(), e);
 		}

src/main/java/com/teamscale/upload/report/xcode/ActionRecord.java

@@ -1,5 +1,9 @@
 package com.teamscale.upload.report.xcode;
 
+import com.google.gson.annotations.JsonAdapter;
+import com.google.gson.annotations.SerializedName;
+import com.teamscale.upload.report.xcode.deserializers.WrappedStringDeserializer;
+
 /**
  * An object of type ActionRecord in the summary JSON output of the XCode
  * xcresulttool executable.
@@ -11,8 +15,16 @@ public class ActionRecord {
 	 */
 	public final ActionResult actionResult;
 
-	public ActionRecord(ActionResult actionResult) {
+	/**
+	 * List of {@link ActionRecord}s.
+	 */
+	@SerializedName("testPlanName")
+	@JsonAdapter(WrappedStringDeserializer.class)
+	public final String testPlanName;
+
+	public ActionRecord(ActionResult actionResult, String testPlanName) {
 		this.actionResult = actionResult;
+		this.testPlanName = testPlanName;
 	}
 
 	/**

src/main/java/com/teamscale/upload/utils/FileSystemUtils.java

@@ -319,22 +319,6 @@ public static void mkdirs(File directory) throws IOException {
 		}
 	}
 
-	/**
-	 * Ensures that the file is an empty file. If the file already exist it is
-	 * deleted and a new empty one is created.
-	 */
-	public static void ensureEmptyFile(File file) throws IOException {
-		if (file.isDirectory()) {
-			throw new IOException("Unable to create empty file because it is a directory: " + file);
-		}
-		if (file.exists() && !file.delete()) {
-			throw new IOException("Unable to delete existing file: " + file);
-		}
-		if (!file.createNewFile()) {
-			throw new IOException("Unable to create file empty file: " + file);
-		}
-	}
-
 	/**
 	 * Recursively delete directories and files. This method ignores the return
 	 * value of delete(), i.e. if anything fails, some files might still exist.

src/main/java/com/teamscale/upload/xcode/ConversionException.java

@@ -0,0 +1,30 @@
+package com.teamscale.upload.xcode;
+
+import com.teamscale.upload.autodetect_revision.ProcessUtils;
+
+/**
+ * Custom exception used to indicate errors during conversion.
+ */
+public class ConversionException extends Exception {
+
+	public ConversionException(String message) {
+		super(message);
+	}
+
+	public ConversionException(String message, Exception e) {
+		super(message, e);
+	}
+
+	/**
+	 * Creates a {@link ConversionException} with the given message and the
+	 * {@linkplain ProcessUtils.ProcessResult#errorOutput error output of the
+	 * command}.
+	 */
+	public static ConversionException withProcessResult(String message, ProcessUtils.ProcessResult processResult) {
+		String messageIncludingErrorOutput = message;
+		if (processResult.errorOutput != null) {
+			messageIncludingErrorOutput += " (command output: " + processResult.errorOutput + ")";
+		}
+		return new ConversionException(messageIncludingErrorOutput, processResult.exception);
+	}
+}

src/main/java/com/teamscale/upload/xcode/ConversionFunction.java

@@ -0,0 +1,11 @@
+package com.teamscale.upload.xcode;
+
+import java.io.IOException;
+
+/** Functional interface that represents a conversion function. */
+@FunctionalInterface
+public interface ConversionFunction<T> {
+
+	/** Runs the conversion function. */
+	T run() throws ConversionException, IOException;
+}

src/main/java/com/teamscale/upload/xcode/ConversionUtils.java

@@ -0,0 +1,69 @@
+package com.teamscale.upload.xcode;
+
+import java.io.File;
+import java.io.IOException;
+
+/** Utilities for converting Xcode coverage reports. */
+public class ConversionUtils {
+	/**
+	 * File extension used for converted XCResult bundles.
+	 */
+	public static final String XCCOV_REPORT_FILE_EXTENSION = ".xccov";
+
+	/**
+	 * File extension used for xccov archives
+	 *
+	 * @see #isXccovArchive(File)
+	 */
+	public static final String XCCOV_ARCHIVE_FILE_EXTENSION = ".xccovarchive";
+
+	/**
+	 * File extension used for xcresult files
+	 *
+	 * @see #isXccovArchive(File)
+	 */
+	public static final String XCRESULT_FILE_EXTENSION = ".xcresult";
+
+	/**
+	 * Returns true if the file is a regular XCResult bundle directory indicated by
+	 * the ".xcresult" ending in the directory name.
+	 */
+	public static boolean isXcresultBundle(File file) {
+		return file.isDirectory() && file.getName().endsWith(XCRESULT_FILE_EXTENSION);
+	}
+
+	/**
+	 * Returns true if the file is a xccov archive which is more compact than a
+	 * regular XCResult bundle. A xccov archive can only be generated by XCode
+	 * internal tooling but provides much better performance when extracting
+	 * coverage. Note that xccov archives don't contain test results.
+	 */
+	public static boolean isXccovArchive(File file) {
+		return file.isDirectory() && file.getName().endsWith(XCCOV_ARCHIVE_FILE_EXTENSION);
+	}
+
+	/**
+	 * Runs the given conversions tasks and ensures that the given teardown is
+	 * executed, even after an interrupt.
+	 */
+	public static <R> R runWithTeardown(ConversionFunction<R> execute, Runnable teardown)
+			throws ConversionException, IOException {
+		Thread cleanupShutdownHook = new Thread(teardown);
+		try {
+			Runtime.getRuntime().addShutdownHook(cleanupShutdownHook);
+
+			return execute.run();
+		} finally {
+			Runtime.getRuntime().removeShutdownHook(cleanupShutdownHook);
+			teardown.run();
+		}
+	}
+
+	/** Removes the suffix from the given string, if present. */
+	public static String removeSuffix(String string, String suffix) {
+		if (string.endsWith(suffix)) {
+			string = string.substring(0, string.length() - suffix.length());
+		}
+		return string;
+	}
+}

src/main/java/com/teamscale/upload/xcode/ConverterBase.java

@@ -0,0 +1,62 @@
+package com.teamscale.upload.xcode;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Base class for class that converts specific file formats into other formats.
+ */
+/* package */ abstract class ConverterBase<T> {
+
+	/**
+	 * The installed and used XCode version. Can be determined by
+	 * {@link XcodeVersion#determine()}.
+	 *
+	 * @implNote The version should be determined by the caller and passed to this
+	 *           class (i.e., instead of determining it in this class). This is
+	 *           because when multiple {@link XcresultConverter} are created, we
+	 *           want to show warnings/errors related to the version determination
+	 *           only once
+	 */
+	private final XcodeVersion xcodeVersion;
+
+	/** The directory where intermediate results can be stored. */
+	private final Path workingDirectory;
+
+	/** The directory where the final conversion results are stored. */
+	private Path outputDirectory;
+
+	public ConverterBase(XcodeVersion xcodeVersion, Path workingDirectory) {
+		this.xcodeVersion = xcodeVersion;
+		this.workingDirectory = workingDirectory;
+	}
+
+	/** Converts a single file into the expected output format. */
+	abstract T convert(File file) throws ConversionException, IOException;
+
+	/**
+	 * Returns the path under which an output file with the given name should be
+	 * stored. Does not create the file though.
+	 */
+	public Path getOutputFilePath(String name) throws IOException {
+		if (outputDirectory == null) {
+			outputDirectory = Files.createTempDirectory(getWorkingDirectory(), this.getClass().getSimpleName());
+		}
+		return outputDirectory.resolve(name);
+	}
+
+	/** Creates a file where the outputs of the converter can be written to. */
+	public File createOutputFile(String name) throws IOException {
+		return Files.createFile(getOutputFilePath(name)).toFile();
+	}
+
+	public XcodeVersion getXcodeVersion() {
+		return xcodeVersion;
+	}
+
+	public Path getWorkingDirectory() {
+		return workingDirectory;
+	}
+}

src/main/java/com/teamscale/upload/xcode/TarArchiveConverter.java

@@ -0,0 +1,44 @@
+package com.teamscale.upload.xcode;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+
+import com.teamscale.upload.utils.FileSystemUtils;
+
+/**
+ * Converts a {@link FileSystemUtils#isTarFile(File) tar file} into the
+ * {@value ConversionUtils#XCCOV_REPORT_FILE_EXTENSION} format.
+ */
+/* package */ class TarArchiveConverter extends ConverterBase<List<File>> {
+
+	public TarArchiveConverter(XcodeVersion xcodeVersion, Path workingDirectory) {
+		super(xcodeVersion, workingDirectory);
+	}
+
+	@Override
+	public List<File> convert(File tarFile) throws ConversionException, IOException {
+		File xcodeReport = extractTar(tarFile);
+
+		if (ConversionUtils.isXccovArchive(xcodeReport)) {
+			return Collections.singletonList(
+					new XccovArchiveConverter(getXcodeVersion(), getWorkingDirectory()).convert(xcodeReport));
+		} else if (ConversionUtils.isXcresultBundle(xcodeReport)) {
+			return new XcresultConverter(getXcodeVersion(), getWorkingDirectory()).convert(xcodeReport);
+		}
+
+		throw new ConversionException(
+				"Report location must be an existing directory with a name that ends with '.xcresult' or "
+						+ "'.xccovarchive'. The directory may be contained in a tar archive indicated by the file "
+						+ "extensions '.tar', '.tar.gz' or '.tgz'." + tarFile);
+	}
+
+	private File extractTar(File tarFile) throws IOException {
+		String tarNameWithoutExtension = FileSystemUtils.stripTarExtension(tarFile.getName());
+		File destination = getWorkingDirectory().resolve(tarNameWithoutExtension).toFile();
+		FileSystemUtils.extractTarArchive(tarFile, destination);
+		return destination;
+	}
+}