Skip to content

Commit

Permalink
adjust compression APIs to be slightly easier to use, and add a basic…
Browse files Browse the repository at this point in the history
… compression snippet
  • Loading branch information
tayloraswift committed Jan 31, 2024
1 parent 8dffe24 commit 7452d1f
Show file tree
Hide file tree
Showing 17 changed files with 107 additions and 70 deletions.
33 changes: 33 additions & 0 deletions Snippets/BasicCompression.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import LZ77

let text:String = """
__,
( o /) _/_
`. , , , , // /
(___)(_(_/_(_ //_ (__
/)
(/
"""

var deflator:LZ77.Deflator = .init(format: .zlib, level: 7, hint: 1_00)
let utf8:[UInt8] = .init(text.utf8)

print("Uncompressed size: \(utf8.count) bytes")

deflator.push(utf8[...], last: false)
deflator.push(utf8[...], last: true)

var data:[UInt8] = []
while let part:[UInt8] = deflator.pull()
{
data += part
}

print("Compressed size: \(data.count) bytes")

var inflator:LZ77.Inflator = .init()
try inflator.push(data[...])

let output:[UInt8] = inflator.pull()

precondition(output == utf8 + utf8, "Round-trip failed!")
24 changes: 19 additions & 5 deletions Sources/LZ77/Deflator/LZ77.Deflator.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ extension LZ77.Deflator
self.stream = .init(format: format, level: level, exponent: e, hint: hint)
self.stream.start(exponent: e)
}

public mutating
func push(_ data:[UInt8], last:Bool = false)
func push(_ data:ArraySlice<UInt8>, last:Bool = false)
{
// rebase input buffer
if !data.isEmpty
Expand All @@ -50,14 +51,27 @@ extension LZ77.Deflator
self.stream.checksum()
}
}

/// Returns a block of compressed data from this deflator, if available. If no compressed
/// data blocks have been completed yet, this method flushes and returns the incomplete
/// block.
public mutating
func pop() -> [UInt8]?
func pull() -> [UInt8]?
{
self.stream.output.pop()
if let complete:[UInt8] = self.pop()
{
return complete
}

let flushed:[UInt8] = self.stream.output.pull()
return flushed.isEmpty ? nil : flushed
}

/// Removes and returns a complete block of compressed data from this deflator, if
/// available.
public mutating
func pull() -> [UInt8]
func pop() -> [UInt8]?
{
return self.pop() ?? self.stream.output.pull()
self.stream.output.pop()
}
}
24 changes: 17 additions & 7 deletions Sources/LZ77/Deflator/LZ77.DeflatorIn.swift
Original file line number Diff line number Diff line change
Expand Up @@ -83,16 +83,26 @@ extension LZ77.DeflatorIn
}

mutating
func enqueue(contentsOf elements:[UInt8])
func enqueue(contentsOf elements:ArraySlice<UInt8>)
{
// always allocate 4 extra tail elements to allow for limited reads
// from beyond the end of the buffer
self.reserve(elements.count + 4)
self.storage.withUnsafeMutablePointerToElements
elements.withUnsafeBufferPointer
{
($0 + self.endIndex).update(from: elements, count: elements.count)
guard
let base:UnsafePointer<UInt8> = $0.baseAddress
else
{
return
}
let count:Int = $0.count
// always allocate 4 extra tail elements to allow for limited reads
// from beyond the end of the buffer
self.reserve(count + 4)
self.storage.withUnsafeMutablePointerToElements
{
($0 + self.endIndex).update(from: base, count: count)
}
self.endIndex += count
}
self.endIndex += elements.count
}

private mutating
Expand Down
2 changes: 1 addition & 1 deletion Sources/LZ77/Deflator/LZ77.DeflatorOut.swift
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ extension LZ77.DeflatorOut
// release all the buffered data chunks. this isn’t ideal (we should
// be releasing them as soon as they are dequeued, but the semantics
// of Array<T> don’t allow for this)
if self.queued <= 0
if self.queued <= 0
{
self.queue.removeAll(keepingCapacity: true)
}
Expand Down
4 changes: 2 additions & 2 deletions Sources/LZ77/Inflator/LZ77.Inflator.Stream.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ extension LZ77.Inflator
struct Stream<Integral> where Integral:LZ77.StreamIntegral
{
// Stream.In manages its own COW in rebase(_:pointer:)
var input:LZ77.InflatorInput
var input:LZ77.InflatorIn
var b:Int
var lengths:[Int]
// Meta and Stream.Out need to have COW manually implemented with
// exclude() on each, to avoid redundant exclusions inside loops,,
// reuse the same buffer since the size is fixed
var meta:LZ77.InflatorTables.Meta
var output:LZ77.InflatorOutput<Integral>
var output:LZ77.InflatorOut<Integral>

#if DUMP_LZ77_BLOCKS || DUMP_LZ77_SYMBOL_HISTOGRAM
// histogram, no match can ever cost more than 17 bits per literal
Expand Down
2 changes: 1 addition & 1 deletion Sources/LZ77/Inflator/LZ77.Inflator.swift
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ extension LZ77.Inflator
{
// returns `nil` if the stream is finished
public mutating
func push(_ data:[UInt8]) throws -> Void?
func push(_ data:ArraySlice<UInt8>) throws -> Void?
{
self.stream.input.rebase(data, pointer: &self.stream.b)
while let _:Void = try self.advance()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
extension LZ77
{
@frozen public
struct InflatorInput
struct InflatorIn
{
private
var capacity:Int, // units in atoms
Expand All @@ -15,7 +15,7 @@ extension LZ77
// atom 1 32 [ ← ← ← ← ← ← ← ← ] 16
// atom 2 48 [ ← ← ← ← ← ← ← ← ] 32
// atom 3 64 [ ← ← ← ← ← ← ← ← ] 48
init(_ data:[UInt8])
init(_ data:ArraySlice<UInt8>)
{
self.capacity = 0
self.bytes = 0
Expand All @@ -26,7 +26,7 @@ extension LZ77
}
}
}
extension LZ77.InflatorInput
extension LZ77.InflatorIn
{
var count:Int
{
Expand All @@ -44,10 +44,9 @@ extension LZ77.InflatorInput

/// Discards all bits before the pointer `b`
mutating
func rebase(_ data:[UInt8], pointer b:inout Int)
func rebase(_ data:ArraySlice<UInt8>, pointer b:inout Int)
{
guard !data.isEmpty
else
if data.isEmpty
{
return
}
Expand Down Expand Up @@ -192,11 +191,11 @@ extension LZ77.InflatorInput
}
}
}
extension LZ77.InflatorInput:ExpressibleByArrayLiteral
extension LZ77.InflatorIn:ExpressibleByArrayLiteral
{
public
init(arrayLiteral:UInt8...)
{
self.init(arrayLiteral)
self.init(arrayLiteral[...])
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
extension LZ77
{
@frozen @usableFromInline
struct InflatorOutput<Integral> where Integral:LZ77.StreamIntegral
struct InflatorOut<Integral> where Integral:LZ77.StreamIntegral
{
var window:Int

Expand Down Expand Up @@ -35,7 +35,7 @@ extension LZ77
}
}
}
extension LZ77.InflatorOutput
extension LZ77.InflatorOut
{
mutating
func exclude()
Expand Down
4 changes: 2 additions & 2 deletions Sources/LZ77/Wrappers/LZ77.DeflateFormat.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ extension LZ77.DeflateFormat:LZ77.StreamFormat
typealias Integral = LZ77.MRC32

public
func begin(inflating input:inout LZ77.InflatorInput,
func begin(inflating input:inout LZ77.InflatorIn,
at bit:inout Int) throws -> LZ77.DeflateHeader?
{
if case .ios = self
Expand Down Expand Up @@ -66,7 +66,7 @@ extension LZ77.DeflateFormat:LZ77.StreamFormat
}

public
func check(inflating input:inout LZ77.InflatorInput, at bit:inout Int) -> UInt32??
func check(inflating input:inout LZ77.InflatorIn, at bit:inout Int) -> UInt32??
{
if case .ios = self
{
Expand Down
4 changes: 2 additions & 2 deletions Sources/LZ77/Wrappers/LZ77.StreamFormat.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ protocol _LZ77StreamFormat
associatedtype Integral:LZ77.StreamIntegral
associatedtype Header:LZ77.StreamHeader

func begin(inflating input:inout LZ77.InflatorInput, at bit:inout Int) throws -> Header?
func check(inflating input:inout LZ77.InflatorInput, at bit:inout Int) -> UInt32??
func begin(inflating input:inout LZ77.InflatorIn, at bit:inout Int) throws -> Header?
func check(inflating input:inout LZ77.InflatorIn, at bit:inout Int) -> UInt32??
}
2 changes: 1 addition & 1 deletion Sources/LZ77Tests/Main.Bitstreams.swift
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ extension Main.Bitstreams:TestBattery
{
if let tests:TestGroup = tests / "Decode"
{
var bits:LZ77.InflatorInput =
var bits:LZ77.InflatorIn =
[
0b1001_1110,
0b1111_0110,
Expand Down
13 changes: 4 additions & 9 deletions Sources/LZ77Tests/Main.Compression.swift
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,16 @@ extension Main.Compression:TestBattery
let input:[UInt8] = (0 ..< count).map{ _ in .random(in: .min ... .max) }

var deflator:LZ77.Deflator = .init(level: level, exponent: 8, hint: 16)
deflator.push(input, last: true)
deflator.push(input[...], last: true)

var compressed:[UInt8] = []
while true
while let part:[UInt8] = deflator.pull()
{
let part:[UInt8] = deflator.pull()
if part.isEmpty
{
break
}
compressed.append(contentsOf: part)
compressed += part
}

var inflator:LZ77.Inflator = .init()
try inflator.push(compressed)
try inflator.push(compressed[...])

let output:[UInt8] = inflator.pull()

Expand Down
2 changes: 1 addition & 1 deletion Sources/LZ77Tests/Main.Matching.swift
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ extension Main.Matching:TestBattery
var output:[[UInt8]] = []
for (s, segment):(Int, [UInt8]) in segments.enumerated()
{
input.enqueue(contentsOf: segment)
input.enqueue(contentsOf: segment[...])

let lookahead:Int = (s == segments.count - 1 ? 0 : 10)
while window.endIndex < 0, input.count > lookahead
Expand Down
2 changes: 1 addition & 1 deletion Sources/PNG/Decoding/PNG.Decoder.swift
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ extension PNG.Decoder
throw PNG.DecodingError.extraneousImageDataCompressedData
}

self.continue = try self.inflator.push(data)
self.continue = try self.inflator.push(data[...])

let delay:Int = (pixel.volume + 7) >> 3
if let pass:Int = self.pass
Expand Down
14 changes: 7 additions & 7 deletions Sources/PNG/Encoding/PNG.Encoder.swift
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ extension PNG.Encoder
}

mutating
func pull(size:(x:Int, y:Int), pixel:PNG.Format.Pixel,
func pull(size:(x:Int, y:Int),
pixel:PNG.Format.Pixel,
delegate:(inout UnsafeMutableBufferPointer<UInt8>, (x:Int, y:Int), Int) throws -> ())
rethrows -> [UInt8]?
{
Expand Down Expand Up @@ -70,7 +71,7 @@ extension PNG.Encoder
self.row = nil
for y:Int in start ..< subimage.y
{
if let data:[UInt8] = self.deflator.pop()
if let data:[UInt8] = self.deflator.pop()
{
self.row = (y, last)
self.pass = .subimage(z)
Expand All @@ -87,7 +88,7 @@ extension PNG.Encoder
$1 = last.count
}

self.deflator.push(Self.filter(scanline, last: last, delay: delay))
self.deflator.push(Self.filter(scanline, last: last, delay: delay)[...])
last = scanline
}
}
Expand All @@ -103,7 +104,7 @@ extension PNG.Encoder
self.row = nil
for y:Int in start ..< size.y
{
if let data:[UInt8] = self.deflator.pop()
if let data:[UInt8] = self.deflator.pop()
{
self.row = (y, last)
return data
Expand All @@ -119,7 +120,7 @@ extension PNG.Encoder
$1 = last.count
}

self.deflator.push(Self.filter(scanline, last: last, delay: delay))
self.deflator.push(Self.filter(scanline, last: last, delay: delay)[...])
last = scanline
}

Expand All @@ -130,8 +131,7 @@ extension PNG.Encoder
break
}

let data:[UInt8] = self.deflator.pull()
return data.isEmpty ? nil : data
return self.deflator.pull()
}

static
Expand Down
13 changes: 3 additions & 10 deletions Sources/PNG/Parsing/PNG.ColorProfile.swift
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ extension PNG.ColorProfile
}

var inflator:LZ77.Inflator = .init()
guard try inflator.push(.init(data.dropFirst(k + 2))) == nil
guard case nil = try inflator.push(data.dropFirst(k + 2))
else
{
throw PNG.ParsingError.incompleteColorProfileCompressedDatastream
Expand All @@ -108,16 +108,9 @@ extension PNG.ColorProfile
data.append(0) // compression method

var deflator:LZ77.Deflator = .init(level: 13, exponent: 15, hint: 4096)
deflator.push(self.profile, last: true)
while true
deflator.push(self.profile[...], last: true)
while let segment:[UInt8] = deflator.pull()
{
let segment:[UInt8] = deflator.pull()
guard !segment.isEmpty
else
{
break
}

data.append(contentsOf: segment)
}

Expand Down
Loading

0 comments on commit 7452d1f

Please sign in to comment.