Simple quality measurement tool - Java adaptation

src/main/org/audiveris/omr/CLI.java

@@ -927,7 +927,7 @@ protected void processBook (Book book)
                 LogUtil.stopBook();
 
                 if (OMR.gui == null) {
-                    LogUtil.removeAppender(book.getRadix());
+                    LogUtil.stopAndRemoveAppender(book.getRadix());
                 }
             }
         }

src/main/org/audiveris/omr/log/LogUtil.java

@@ -314,10 +314,11 @@ private static void initMessage (String str)
      *
      * @param name appender name (typically the book radix)
      */
-    public static void removeAppender (String name)
+    public static void stopAndRemoveAppender (String name)
     {
         Logger root = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(
                 Logger.ROOT_LOGGER_NAME);
+        root.getAppender(name).stop();
         root.detachAppender(name);
     }
 

src/main/org/audiveris/omr/util/FileUtil.java

@@ -43,6 +43,8 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
+import java.util.function.Predicate;
 
 /**
  * Class <code>FileUtil</code> gathers convenient utility methods for files (and paths).
@@ -486,4 +488,38 @@ public FileVisitResult visitFile (Path file,
 
         return pathsFound;
     }
+
+    //---------------------//
+    // findFileInDirectory //
+    //---------------------//
+    /**
+     * Finds a file in the given directory (non-recursive) matching the given predicate.
+     *
+     * @param directory   the directory in which to look for the file
+     * @param fileMatches the condition for inclusion, evaluated for each file
+     * @return the first matching file
+     */
+    public static Optional<Path> findFileInDirectory (Path directory, Predicate<Path> fileMatches)
+            throws IOException
+    {
+        return Files.walk(directory, 1)
+                .filter(Files::isRegularFile)
+                .filter(fileMatches)
+                .findFirst();
+    }
+
+    //---------------------------------//
+    // fileNameWithoutExtensionMatches //
+    //---------------------------------//
+    /**
+     * Creates a predicate on Path that evaluates to true if the path's name matches the given file
+     * name. To be used e.g. as parameter to {@link #findFileInDirectory(Path, Predicate)}.
+     *
+     * @param fileName the file name to match
+     * @return the predicate
+     */
+    public static Predicate<Path> fileNameWithoutExtensionMatches (String fileName)
+    {
+        return path -> FileUtil.sansExtension(path.getFileName().toString()).equals(fileName);
+    }
 }

src/main/org/audiveris/omr/util/SetOperation.java

@@ -0,0 +1,72 @@
+//------------------------------------------------------------------------------------------------//
+//                                                                                                //
+//                                    S e t O p e r a t i o n                                     //
+//                                                                                                //
+//------------------------------------------------------------------------------------------------//
+// <editor-fold defaultstate="collapsed" desc="hdr">
+//
+//  Copyright © Audiveris 2022. All rights reserved.
+//
+//  This program is free software: you can redistribute it and/or modify it under the terms of the
+//  GNU Affero General Public License as published by the Free Software Foundation, either version
+//  3 of the License, or (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+//  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//  See the GNU Affero General Public License for more details.
+//
+//  You should have received a copy of the GNU Affero General Public License along with this
+//  program.  If not, see <http://www.gnu.org/licenses/>.
+//------------------------------------------------------------------------------------------------//
+// </editor-fold>
+package org.audiveris.omr.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Utility class for operations on Sets.
+ *
+ * @author Peter Greth
+ */
+public abstract class SetOperation
+{
+    /**
+     * The union of two sets.
+     * Returns all elements that are in any of the two sets.
+     */
+    public static <T> Set<T> union (Set<T> a, Set<T> b)
+    {
+        Set<T> unionSet = new HashSet<>();
+        unionSet.addAll(a);
+        unionSet.addAll(b);
+        return unionSet;
+    }
+
+    /**
+     * The intersection of two sets.
+     * Returns only those elements that are in both of the sets.
+     */
+    @SuppressWarnings("CollectionAddAllCanBeReplacedWithConstructor")
+    public static <T> Set<T> intersection (Set<T> a, Set<T> b)
+    {
+        Set<T> unionSet = new HashSet<>();
+        unionSet.addAll(a);
+        unionSet.retainAll(b);
+        return unionSet;
+    }
+
+    /**
+     * The diff of two sets.
+     * Returns only those elements of set a that are not in set b.
+     */
+    @SuppressWarnings("CollectionAddAllCanBeReplacedWithConstructor")
+    public static <T> Set<T> diff (Set<T> a, Set<T> b)
+    {
+        Set<T> diffSet = new HashSet<>();
+        diffSet.addAll(a);
+        diffSet.removeAll(b);
+        return diffSet;
+    }
+
+}

src/test/conversion/score/ConversionScoreRegressionTest.java

@@ -0,0 +1,228 @@
+//------------------------------------------------------------------------------------------------//
+//                                                                                                //
+//                    C o n v e r s i o n S c o r e R e g r e s s i o n T e s t                   //
+//                                                                                                //
+//------------------------------------------------------------------------------------------------//
+// <editor-fold defaultstate="collapsed" desc="hdr">
+//
+//  Copyright © Audiveris 2021. All rights reserved.
+//
+//  This program is free software: you can redistribute it and/or modify it under the terms of the
+//  GNU Affero General Public License as published by the Free Software Foundation, either version
+//  3 of the License, or (at your option) any later version.
+//
+//  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+//  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//  See the GNU Affero General Public License for more details.
+//
+//  You should have received a copy of the GNU Affero General Public License along with this
+//  program.  If not, see <http://www.gnu.org/licenses/>.
+//------------------------------------------------------------------------------------------------//
+// </editor-fold>
+package conversion.score;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import org.audiveris.omr.Main;
+import org.audiveris.omr.util.FileUtil;
+import org.audiveris.proxymusic.ScorePartwise;
+import org.audiveris.proxymusic.mxl.Mxl;
+import org.audiveris.proxymusic.util.Marshalling;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.JAXBException;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.zip.ZipEntry;
+
+import static org.audiveris.omr.OMR.COMPRESSED_SCORE_EXTENSION;
+import static org.audiveris.omr.OMR.SCORE_EXTENSION;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Regression test that checks Audiveris' accuracy according to samples located in
+ * src/test/resources/conversion/score.
+ * Similarity of converted input and expected output is measured using class
+ * {@link ScoreSimilarity}.
+ * To add a new sample, please include its directory name and the resulting conversion score in
+ * {@link #TEST_CASES}.
+ *
+ * @author Peter Greth
+ */
+@RunWith(Parameterized.class)
+public class ConversionScoreRegressionTest
+{
+    /**
+     * List of test cases. Each of these will be tested separately
+     */
+    private final static List<ConversionScoreTestCase> TEST_CASES = List.of(
+            ConversionScoreTestCase.ofSubDirectory("01-klavier").withExpectedConversionScore(15)
+    );
+
+    /**
+     * The name of the input file for each test case
+     */
+    private final static String INPUT_FILE_NAME = "input";
+
+    /**
+     * The current test case (provided by {@link #testCaseProvider()}, executed separately by JUnit)
+     */
+    @Parameter
+    public ConversionScoreTestCase underTest;
+
+    /**
+     * The directory into which Audiveris can output the parsed score
+     */
+    private Path outputDirectory;
+
+    /**
+     * @return a list of test cases that are executed each
+     */
+    // "{0}" provides a readable test name using conversion.score.TestCase::toString
+    @Parameters(name = "{0}")
+    public static Collection<ConversionScoreTestCase> testCaseProvider ()
+    {
+        return TEST_CASES;
+    }
+
+    /**
+     * Reduce logging verbosity (increases execution time significantly)
+     */
+    @BeforeClass
+    public static void reduceLoggingVerbosity ()
+    {
+        Logger root = (ch.qos.logback.classic.Logger)
+                LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+        root.setLevel(Level.WARN);
+    }
+
+    @Before
+    public void createOutputDirectory ()
+            throws IOException
+    {
+        String tempDirectoryName = String.format("audiveris-test-%s", underTest.subDirectoryName);
+        outputDirectory = Files.createTempDirectory(tempDirectoryName);
+    }
+
+    @After
+    public void removeOutputDirectory ()
+            throws IOException
+    {
+        if (Files.exists(outputDirectory)) {
+            FileUtil.deleteDirectory(outputDirectory);
+        }
+    }
+
+    /**
+     * The actual regression test, executed for each test case defined in {@link #TEST_CASES}.
+     * Converts the input file using Audiveris, then compares the produced result with the expected
+     * result.
+     * For comparison,
+     * {@link ScoreSimilarity#conversionScore(ScorePartwise.Part, ScorePartwise.Part)} is used.
+     * This test will fail if the conversion score changed in any direction.
+     */
+    @Test
+    public void testConversionScoreChanged ()
+            throws IOException,
+                   JAXBException,
+                   Mxl.MxlException,
+                   Marshalling.UnmarshallingException
+    {
+        ScorePartwise expectedScore = loadXmlScore(underTest.findExpectedOutputFile());
+        assertFalse(String.format("Could not load expected output at '%s': Contains no Part",
+                                  underTest.findExpectedOutputFile()),
+                    expectedScore.getPart().isEmpty());
+
+        Path outputMxl = audiverisBatchExport(outputDirectory, underTest.findInputFile());
+        ScorePartwise actualScore = loadMxlScore(outputMxl);
+        assertFalse(String.format("Could not load actual output in '%s': Contains no Part",
+                                  outputDirectory),
+                    actualScore.getPart().isEmpty());
+
+        int actualConversionScore = ScoreSimilarity.conversionScore(expectedScore, actualScore);
+        failIfConversionScoreDecreased(underTest.expectedConversionScore, actualConversionScore);
+        failIfConversionScoreIncreased(underTest.expectedConversionScore, actualConversionScore);
+    }
+
+    private static ScorePartwise loadXmlScore (Path xmlFile)
+            throws IOException,
+                   Marshalling.UnmarshallingException
+    {
+        try (InputStream inputStream = new FileInputStream(xmlFile.toFile())) {
+            Object score = Marshalling.unmarshal(inputStream);
+            assertTrue(String.format("The parsed xml in '%s' is not of type ScorePartwise",
+                                     xmlFile),
+                       score instanceof ScorePartwise);
+            return (ScorePartwise) score;
+        }
+    }
+
+    private static Path audiverisBatchExport (Path outputDirectory, Path inputFile)
+    {
+        Main.main(new String[]{
+                "-batch",
+                "-export",
+                "-output", outputDirectory.toAbsolutePath().toString(),
+                inputFile.toAbsolutePath().toString()
+        });
+        Path outputMxlFile = outputDirectory
+                .resolve(INPUT_FILE_NAME) // folder named equal to input file name
+                .resolve(INPUT_FILE_NAME + COMPRESSED_SCORE_EXTENSION);
+        assertTrue(String.format("Audiveris batch export seems to have failed. Cannot find " +
+                                         "output file '%s'", outputMxlFile),
+                   Files.exists(outputMxlFile));
+        return outputMxlFile;
+    }
+
+    private static ScorePartwise loadMxlScore (Path mxlFile)
+            throws Mxl.MxlException,
+                   Marshalling.UnmarshallingException,
+                   JAXBException,
+                   IOException
+    {
+        try (Mxl.Input outputMxlFileReader = new Mxl.Input(mxlFile.toFile())) {
+            ZipEntry xmlEntry = outputMxlFileReader.getEntry(INPUT_FILE_NAME + SCORE_EXTENSION);
+            Object score = Marshalling.unmarshal(outputMxlFileReader.getInputStream(xmlEntry));
+            assertTrue(String.format("The parsed mxl in '%s' is not of type ScorePartwise",
+                                     mxlFile),
+                       score instanceof ScorePartwise);
+            return (ScorePartwise) score;
+        }
+    }
+
+    private static void failIfConversionScoreDecreased (int expectedConversionScore,
+                                                        int actualConversionScore)
+    {
+        String message = String.format("The conversion score decreased from %d to %d (diff: %d).",
+                                       expectedConversionScore,
+                                       actualConversionScore,
+                                       actualConversionScore - expectedConversionScore);
+        assertFalse(message, actualConversionScore < expectedConversionScore);
+    }
+
+    private static void failIfConversionScoreIncreased (int expectedConversionScore,
+                                                        int actualConversionScore)
+    {
+        String message = String.format("Well done, the conversion score increased from %d to %d " +
+                                               "(diff: %d). Please adapt " +
+                                               "conversion.score.TestCase::TEST_CASES accordingly.",
+                                       expectedConversionScore,
+                                       actualConversionScore,
+                                       actualConversionScore - expectedConversionScore);
+        assertFalse(message, actualConversionScore > expectedConversionScore);
+    }
+
+}