Compressed music format

helper/elf2pomf.py

@@ -3,6 +3,22 @@
 from elftools.elf.elffile import ELFFile
 from sys import argv
 import pdb
+import zlib
+
+FS_BLKSZ = 4096
+
+def compress_bytes(data, level = 5):
+    assert(zlib.MAX_WBITS == 15)
+    compress = zlib.compressobj(
+        level,
+        zlib.DEFLATED,
+        -zlib.MAX_WBITS,
+        zlib.DEF_MEM_LEVEL,
+        0
+    )
+    rslt = compress.compress(data)
+    rslt += compress.flush()
+    return rslt
 
 INNAME=argv[1]
 OUTNAME=argv[2]
@@ -66,17 +82,42 @@
         track_data[offset + 3] = idx_le[3]
     cur_sample_index += 1
 
+def create_pomf(compressed):
+    global samples_indexed, track_data, header
+    out = bytearray(header)
+    for s in samples_indexed:
+        sdata = bytearray(s['header']) + bytearray(s['data'])
+        orig_size_le = len(sdata).to_bytes(4, byteorder = 'little')
+        if compressed:
+            sdata = compress_bytes(sdata)
+        size_le = len(sdata).to_bytes(4, byteorder = 'little')
+        out += (b'saZZ' if compressed else b'saMP')
+        out += (size_le)
+        out += (orig_size_le)
+        out += (sdata)
+    orig_size_le = (len(track_data)).to_bytes(4, byteorder = 'little')
+    if compressed:
+        track_data = compress_bytes(track_data)
+    size_le = (len(track_data)).to_bytes(4, byteorder = 'little')
+    out += (b'tuZZ' if compressed else b'tuNE')
+    out += (size_le)
+    out += (orig_size_le)
+    out += (track_data)
+    out += (b'eof EoF EOF ')
+    return out
+
+uncomp = create_pomf(False)
+comp = create_pomf(True)
+
+sz_uncomp_blocks = len(uncomp) // FS_BLKSZ + 1
+sz_comp_blocks = len(comp) // FS_BLKSZ + 1
+
 outf = open(OUTNAME, 'wb')
 
-outf.write(header)
-for s in samples_indexed:
-    outf.write(b'saMP')
-    size_le = (len(s['header']) + len(s['data'])).to_bytes(4, byteorder = 'little')
-    outf.write(size_le)
-    outf.write(s['header'])
-    outf.write(s['data'])
-outf.write(b'tuNE')
-size_le = (len(track_data)).to_bytes(4, byteorder = 'little')
-outf.write(size_le)
-outf.write(track_data)
-outf.write(b'eof \x00\x00\x00\x00')
+if sz_comp_blocks < sz_uncomp_blocks:
+    print("Compression saves",(sz_uncomp_blocks - sz_comp_blocks),"of 4K FS blocks: from",sz_uncomp_blocks,"to",sz_comp_blocks)
+    outf.write(comp)
+else:
+    outf.write(uncomp)
+
+outf.close()

include/miniz_ext.h

@@ -0,0 +1,54 @@
+#include <rom/miniz.h>
+
+// MiniZ extra functions that are not present in the ROM
+
+
+// Flush values. For typical usage you only need MZ_NO_FLUSH and MZ_FINISH. The other values are for advanced use (refer to the zlib docs).
+enum { MZ_NO_FLUSH = 0, MZ_PARTIAL_FLUSH = 1, MZ_SYNC_FLUSH = 2, MZ_FULL_FLUSH = 3, MZ_FINISH = 4, MZ_BLOCK = 5 };
+
+// Return status codes. MZ_PARAM_ERROR is non-standard.
+enum { MZ_OK = 0, MZ_STREAM_END = 1, MZ_NEED_DICT = 2, MZ_ERRNO = -1, MZ_STREAM_ERROR = -2, MZ_DATA_ERROR = -3, MZ_MEM_ERROR = -4, MZ_BUF_ERROR = -5, MZ_VERSION_ERROR = -6, MZ_PARAM_ERROR = -10000 };
+
+// Compression levels: 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow), MZ_DEFAULT_COMPRESSION=MZ_DEFAULT_LEVEL.
+enum { MZ_NO_COMPRESSION = 0, MZ_BEST_SPEED = 1, MZ_BEST_COMPRESSION = 9, MZ_UBER_COMPRESSION = 10, MZ_DEFAULT_LEVEL = 6, MZ_DEFAULT_COMPRESSION = -1 };
+
+// Window bits
+#define MZ_DEFAULT_WINDOW_BITS 15
+
+// Heap allocation callbacks.
+// Note that mz_alloc_func parameter types purpsosely differ from zlib's: items/size is size_t, not unsigned long.
+typedef void *(*mz_alloc_func)(void *opaque, size_t items, size_t size);
+typedef void (*mz_free_func)(void *opaque, void *address);
+typedef void *(*mz_realloc_func)(void *opaque, void *address, size_t items, size_t size);
+
+struct mz_internal_state;
+
+// Compression/decompression stream struct.
+typedef struct mz_stream_s {
+    const unsigned char *next_in;     // pointer to next byte to read
+    unsigned int avail_in;            // number of bytes available at next_in
+    mz_ulong total_in;                // total number of bytes consumed so far
+
+    unsigned char *next_out;          // pointer to next byte to write
+    unsigned int avail_out;           // number of bytes that can be written to next_out
+    mz_ulong total_out;               // total number of bytes produced so far
+
+    char *msg;                        // error msg (unused)
+    struct mz_internal_state *state;  // internal state, allocated by zalloc/zfree
+
+    mz_alloc_func zalloc;             // optional heap allocation function (defaults to malloc)
+    mz_free_func zfree;               // optional heap free function (defaults to free)
+    void *opaque;                     // heap alloc function user pointer
+
+    int data_type;                    // data_type (unused)
+    mz_ulong adler;                   // adler32 of the source or uncompressed data
+    mz_ulong reserved;                // not used
+} mz_stream;
+
+typedef mz_stream *mz_streamp;
+
+/* Single-call decompression. */
+/* Returns MZ_OK on success, or one of the error codes from mz_inflate() on failure. */
+int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len);
+int mz_uncompress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong *pSource_len);
+const char *mz_error(int err);

include/sound/pomf.h

@@ -27,22 +27,32 @@ Format in a nutshell:
                 - rle_sample_t followed by .length bytes of sample data
             - if magic is 'tuNE':
                 - .size bytes of melody_item_t data
+            - if magic is 'saZZ' or 'tuZZ':
+                - contents of 'saMP' or 'tuNE' respectively, compressed with DEFLATE with a -15 bit window.
+                  `size` shows the length of the compressed data, `realsize` the length of the target buffer for uncompression.
     ]
     POMFChunk(size = 0, magic = ASCII 'eof ')
+
+Version history:
+    1.0: Initial version
+    1.1: Add primitive compression support (saZZ, tuZZ chunks)
  */
 
 #define POMF_MAGIC_FILE 0x666D4F50 // 'POmf' little-endian
 #define POMF_MAGIC_SAMPLE 0x504D6173 // 'saMP'
+#define POMF_MAGIC_SAMPLE_COMPRESSED 0x5A5A6173 // 'saZZ'
 #define POMF_MAGIC_TRACK 0x454E7574 // 'tuNE'
+#define POMF_MAGIC_TRACK_COMPRESSED 0x5A5A7574 // 'tuZZ'
 #define POMF_MAGIC_END 0x20666F65 // 'eof '
 
 #define POMF_CURVER_MAJ (1)
-#define POMF_CURVER_MIN (0)
+#define POMF_CURVER_MIN (1)
 #define POMF_CURVER ((POMF_CURVER_MAJ << 8) | (POMF_CURVER_MIN))
 
 struct POMFChunkHeader {
     const uint32_t magic;
     const uint32_t size;
+    const uint32_t realsize;
 };
 
 struct __attribute__((packed)) POMFHeader {