Skip to content

Commit 022cad7

Browse files
authored
Revert test addition in CompressionCodecSuite
Was a duplicate of the Snappy test instead of exercising the problem from Spark, something to think about later
1 parent accc88f commit 022cad7

1 file changed

Lines changed: 1 addition & 55 deletions

File tree

core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala

Lines changed: 1 addition & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.io
1919

20-
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream}
20+
import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
2121

2222
import com.google.common.io.ByteStreams
2323

@@ -130,58 +130,4 @@ class CompressionCodecSuite extends SparkFunSuite {
130130
ByteStreams.readFully(concatenatedBytes, decompressed)
131131
assert(decompressed.toSeq === (0 to 127))
132132
}
133-
134-
// Based on https://github.com/xerial/snappy-java/blob/60cc0c2e1d1a76ae2981d0572a5164fcfdfba5f1/src/test/java/org/xerial/snappy/SnappyInputStreamTest.java
135-
test("SPARK 17378: snappy-java should handle magic header when reading stream") {
136-
val b = new ByteArrayOutputStream()
137-
// Write uncompressed length beginning with -126 (the same with magicheader[0])
138-
b.write(-126) // Can't access magic header[0] as it isn't public, so access this way
139-
b.write(0x01)
140-
// uncompressed data length = 130
141-
142-
var data = new ByteArrayOutputStream()
143-
144-
for (i <- 0 until 130) {
145-
data.write('A')
146-
}
147-
148-
var dataMoreThan8Len = data.toByteArray()
149-
150-
// write literal (lower 2-bit of the first tag byte is 00, upper 6-bits represents data size)
151-
b.write(60<<2) // 1-byte data length follows
152-
b.write(dataMoreThan8Len.length-1) // subsequent data length
153-
b.write(dataMoreThan8Len)
154-
155-
var compressed = b.toByteArray()
156-
157-
// This should succeed
158-
assert(dataMoreThan8Len === org.xerial.snappy.Snappy.uncompress(compressed))
159-
160-
// Reproduce error in #142
161-
val in = new org.xerial.snappy.SnappyInputStream(new ByteArrayInputStream(b.toByteArray()))
162-
163-
var uncompressed = readFully(in)
164-
assert(dataMoreThan8Len === uncompressed) // this fails as uncompressed is empty
165-
}
166-
167-
private def readFully(input: InputStream): Array[Byte] = {
168-
try {
169-
val out = new ByteArrayOutputStream()
170-
var buf = new Array[Byte](4096)
171-
172-
var readBytes = 0
173-
while (readBytes != -1) {
174-
readBytes = input.read(buf)
175-
if (readBytes != -1) {
176-
out.write(buf, 0, readBytes)
177-
}
178-
}
179-
out.flush()
180-
return out.toByteArray()
181-
}
182-
finally {
183-
input.close();
184-
}
185-
}
186-
187133
}

0 commit comments

Comments
 (0)