zstd-ruby/spec/streaming_128kb_cliff_spec.rb at f25d76eed78c10ef98ad64dcb9144232bad5e29c · SpringMT/zstd-ruby · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
# spec/streaming_128kb_cliff_spec.rb
require "spec_helper"
require "stringio"
require "zstd-ruby"

RSpec.describe "Zstd streaming 128KB cliff" do
  MAGIC = [0x28, 0xB5, 0x2F, 0xFD].freeze

  def hex(str)
    str.unpack1("H*")
  end

  def has_magic?(bin)
    bytes = bin.bytes
    bytes[0, 4] == MAGIC
  end

  shared_examples "round-trip streaming compress" do |size|
    it "round-trips #{size} bytes and starts with zstd magic" do
      # Produce data
      src = "a" * size

      sc = Zstd::StreamingCompress.new
      sc << src
      compressed = sc.finish

      expect(has_magic?(compressed)).to be(true), "missing magic: #{hex(compressed)[0, 16]}"
      expect(Zstd.decompress(compressed)).to eq(src)
    end
  end

  context "exactly around 128 KiB" do
    include_examples "round-trip streaming compress", 131_071
    include_examples "round-trip streaming compress", 131_072     # the cliff
    include_examples "round-trip streaming compress", 131_073
  end

  context "multiple writes crossing the boundary" do
    it "64KiB + 64KiB (exact boundary) works" do
      part = "x" * 65_536
      sc = Zstd::StreamingCompress.new
      sc << part
      sc << part
      compressed = sc.finish
      expect(has_magic?(compressed)).to be(true)
      expect(Zstd.decompress(compressed)).to eq(part * 2)
    end

    it "64KiB + 64KiB + 1 works" do
      a = "a" * 65_536
      b = "b" * 65_536
      c = "c"
      sc = Zstd::StreamingCompress.new
      sc << a << b << c
      compressed = sc.finish
      expect(has_magic?(compressed)).to be(true)
      expect(Zstd.decompress(compressed)).to eq(a + b + c)
    end
  end

  context "flush/end draining" do
    it "returns all produced bytes across flush and finish" do
      sc = Zstd::StreamingCompress.new
      sc << ("a" * 70_000)
      out = sc.flush
      expect(out).not_to be_empty
      sc << ("b" * 70_000)
      out << sc.finish
      expect(has_magic?(out)).to be(true)
      expect(Zstd.decompress(out)).to eq(("a" * 70_000) + ("b" * 70_000))
    end
  end

  context "GC.compact interaction" do
    it "survives compaction around the boundary" do
      sc = Zstd::StreamingCompress.new
      sc << ("a" * 80_000)
      GC.compact if GC.respond_to?(:compact)
      sc << ("b" * 60_000) # total now > 128KiB
      compressed = sc.finish
      expect(has_magic?(compressed)).to be(true)
      expect(Zstd.decompress(compressed)).to eq(("a" * 80_000) + ("b" * 60_000))
    end
  end

  context "larger payload sanity" do
    it "round-trips ~1 MiB" do
      src = "z" * 1_048_576
      sc = Zstd::StreamingCompress.new(level: 3)
      sc << src
      compressed = sc.finish
      expect(has_magic?(compressed)).to be(true)
      expect(Zstd.decompress(compressed)).to eq(src)
    end
  end

  describe Zstd::StreamWriter do
    it "produces a valid frame and round-trips at exactly 128KiB" do
      io = StringIO.new
      sw = Zstd::StreamWriter.new(io)
      sw.write("a" * 131_072)
      sw.finish

      io.rewind
      bin = io.read
      expect(has_magic?(bin)).to be(true), "missing magic: #{hex(bin)[0, 16]}"

      io.rewind
      sr = Zstd::StreamReader.new(io)
      out = sr.read(2_000_000)
      expect(out.size).to eq(131_072)
      expect(out).to eq("a" * 131_072)
      io.close
    end

    it "handles boundary-crossing writes with flush in between" do
      io = StringIO.new
      sw = Zstd::StreamWriter.new(io)
      sw.write("a" * 70_000)
      sw.write("b" * 70_000) # crosses 128KiB internally
      sw.finish

      io.rewind
      bin = io.read
      expect(has_magic?(bin)).to be(true)
      io.rewind
      sr = Zstd::StreamReader.new(io)
      out = sr.read(1_000_000)
      expect(out).to eq("a" * 70_000 + "b" * 70_000)
      io.close
    end

    it "survives GC.compact mid-stream" do
      io = StringIO.new
      sw = Zstd::StreamWriter.new(io)
      sw.write("x" * 90_000)
      GC.compact if GC.respond_to?(:compact)
      sw.write("y" * 50_000)
      sw.finish

      io.rewind
      bin = io.read
      expect(has_magic?(bin)).to be(true)
      io.rewind
      sr = Zstd::StreamReader.new(io)
      out = sr.read(200_000)
      expect(out).to eq("x" * 90_000 + "y" * 50_000)
    end
  end
end