diff --git a/libraries/common/src/main/java/androidx/media3/common/FileTypes.java b/libraries/common/src/main/java/androidx/media3/common/FileTypes.java index 38324d2f1ec..6fa919133c5 100644 --- a/libraries/common/src/main/java/androidx/media3/common/FileTypes.java +++ b/libraries/common/src/main/java/androidx/media3/common/FileTypes.java @@ -59,6 +59,7 @@ public final class FileTypes { *
  • {@link #BMP} *
  • {@link #HEIF} *
  • {@link #AVIF} + *
  • {@link #MJPEG} * */ @Documented @@ -66,7 +67,7 @@ public final class FileTypes { @Target(TYPE_USE) @IntDef({ UNKNOWN, AC3, AC4, ADTS, AMR, FLAC, FLV, MATROSKA, MP3, MP4, OGG, PS, TS, WAV, WEBVTT, JPEG, - MIDI, AVI, PNG, WEBP, BMP, HEIF, AVIF + MIDI, AVI, PNG, WEBP, BMP, HEIF, AVIF, MJPEG }) public @interface Type {} @@ -139,6 +140,9 @@ public final class FileTypes { /** File type for the AVIF format. */ public static final int AVIF = 21; + /** File type for Motion JPEG streams. */ + public static final int MJPEG = 22; + @VisibleForTesting /* package */ static final String HEADER_CONTENT_TYPE = "Content-Type"; private static final String EXTENSION_AC3 = ".ac3"; @@ -173,6 +177,8 @@ public final class FileTypes { private static final String EXTENSION_WEBVTT = ".webvtt"; private static final String EXTENSION_JPG = ".jpg"; private static final String EXTENSION_JPEG = ".jpeg"; + private static final String EXTENSION_MJPG = ".mjpg"; + private static final String EXTENSION_MJPEG = ".mjpeg"; private static final String EXTENSION_AVI = ".avi"; private static final String EXTENSION_PNG = ".png"; private static final String EXTENSION_WEBP = ".webp"; @@ -187,7 +193,14 @@ private FileTypes() {} /** Returns the {@link Type} corresponding to the response headers provided. */ public static @FileTypes.Type int inferFileTypeFromResponseHeaders( Map> responseHeaders) { - @Nullable List contentTypes = responseHeaders.get(HEADER_CONTENT_TYPE); + @Nullable List contentTypes = null; + for (Map.Entry> responseHeader : responseHeaders.entrySet()) { + if (responseHeader.getKey() != null + && responseHeader.getKey().equalsIgnoreCase(HEADER_CONTENT_TYPE)) { + contentTypes = responseHeader.getValue(); + break; + } + } @Nullable String mimeType = contentTypes == null || contentTypes.isEmpty() ? null : contentTypes.get(0); return inferFileTypeFromMimeType(mimeType); @@ -202,6 +215,10 @@ private FileTypes() {} if (mimeType == null) { return FileTypes.UNKNOWN; } + int parametersStartIndex = mimeType.indexOf(';'); + if (parametersStartIndex != -1) { + mimeType = mimeType.substring(0, parametersStartIndex).trim(); + } mimeType = normalizeMimeType(mimeType); switch (mimeType) { case MimeTypes.AUDIO_AC3: @@ -257,6 +274,9 @@ private FileTypes() {} return FileTypes.HEIF; case MimeTypes.IMAGE_AVIF: return FileTypes.AVIF; + case MimeTypes.MULTIPART_MJPEG: + case MimeTypes.VIDEO_MJPEG: + return FileTypes.MJPEG; default: return FileTypes.UNKNOWN; } @@ -320,6 +340,8 @@ private FileTypes() {} return FileTypes.WAV; } else if (filename.endsWith(EXTENSION_VTT) || filename.endsWith(EXTENSION_WEBVTT)) { return FileTypes.WEBVTT; + } else if (filename.endsWith(EXTENSION_MJPG) || filename.endsWith(EXTENSION_MJPEG)) { + return FileTypes.MJPEG; } else if (filename.endsWith(EXTENSION_JPG) || filename.endsWith(EXTENSION_JPEG)) { return FileTypes.JPEG; } else if (filename.endsWith(EXTENSION_AVI)) { diff --git a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java index 134bafc85fd..d4a58bf2688 100644 --- a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java +++ b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java @@ -195,6 +195,11 @@ public final class MimeTypes { @UnstableApi public static final String IMAGE_WEBP = BASE_TYPE_IMAGE + "/webp"; @UnstableApi public static final String IMAGE_RAW = BASE_TYPE_IMAGE + "/raw"; + // multipart/ MIME types + + /** MIME type for multipart Motion JPEG streams. */ + @UnstableApi public static final String MULTIPART_MJPEG = "multipart/x-mixed-replace"; + /** * A non-standard codec string for E-AC3-JOC. Use of this constant allows for disambiguation * between regular E-AC3 ("ec-3") and E-AC3-JOC ("ec+3") streams from the codec string alone. The diff --git a/libraries/common/src/test/java/androidx/media3/common/FileTypesTest.java b/libraries/common/src/test/java/androidx/media3/common/FileTypesTest.java index f5439da50a1..f8364716427 100644 --- a/libraries/common/src/test/java/androidx/media3/common/FileTypesTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/FileTypesTest.java @@ -42,6 +42,28 @@ public void inferFileFormat_fromResponseHeaders_returnsExpectedFormat() { .isEqualTo(FileTypes.MP4); } + @Test + public void inferFileFormat_fromMjpegResponseHeaders_returnsMjpeg() { + Map> responseHeaders = new HashMap<>(); + responseHeaders.put( + "Content-type", + Collections.singletonList("multipart/x-mixed-replace; boundary=--myboundary")); + + assertThat(FileTypes.inferFileTypeFromResponseHeaders(responseHeaders)) + .isEqualTo(FileTypes.MJPEG); + } + + @Test + public void inferFileFormat_fromMjpegResponseHeadersWithQuotedBoundary_returnsMjpeg() { + Map> responseHeaders = new HashMap<>(); + responseHeaders.put( + "Content-Type", + Collections.singletonList("multipart/x-mixed-replace; boundary=\"--totalmjpeg\"")); + + assertThat(FileTypes.inferFileTypeFromResponseHeaders(responseHeaders)) + .isEqualTo(FileTypes.MJPEG); + } + @Test public void inferFileFormat_fromResponseHeadersWithUnknownContentType_returnsUnknownFormat() { Map> responseHeaders = new HashMap<>(); @@ -62,6 +84,12 @@ public void inferFileFormat_fromMimeType_returnsExpectedFormat() { assertThat(FileTypes.inferFileTypeFromMimeType("audio/x-flac")).isEqualTo(FileTypes.FLAC); } + @Test + public void inferFileFormat_fromMjpegMimeType_returnsMjpeg() { + assertThat(FileTypes.inferFileTypeFromMimeType(MimeTypes.VIDEO_MJPEG)) + .isEqualTo(FileTypes.MJPEG); + } + @Test public void inferFileFormat_fromUnknownMimeType_returnsUnknownFormat() { assertThat(inferFileTypeFromMimeType(/* mimeType= */ "unknown")).isEqualTo(FileTypes.UNKNOWN); @@ -85,6 +113,12 @@ public void inferFileFormat_fromUriWithExtensionPrefix_returnsExpectedFormat() { assertThat(inferFileTypeFromUri(Uri.parse("filename.mka"))).isEqualTo(FileTypes.MATROSKA); } + @Test + public void inferFileFormat_fromMjpegUri_returnsMjpeg() { + assertThat(inferFileTypeFromUri(Uri.parse("filename.mjpeg"))).isEqualTo(FileTypes.MJPEG); + assertThat(inferFileTypeFromUri(Uri.parse("filename.mjpg"))).isEqualTo(FileTypes.MJPEG); + } + @Test public void inferFileFormat_fromUriWithUnknownExtension_returnsUnknownFormat() { assertThat(inferFileTypeFromUri(Uri.parse("filename.unknown"))).isEqualTo(FileTypes.UNKNOWN); diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java b/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java index 03f062508b1..c22a13f6fd4 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java @@ -36,6 +36,7 @@ import androidx.media3.extractor.flv.FlvExtractor; import androidx.media3.extractor.heif.HeifExtractor; import androidx.media3.extractor.jpeg.JpegExtractor; +import androidx.media3.extractor.jpeg.MjpegExtractor; import androidx.media3.extractor.mkv.MatroskaExtractor; import androidx.media3.extractor.mp3.Mp3Extractor; import androidx.media3.extractor.mp4.FragmentedMp4Extractor; @@ -88,6 +89,7 @@ * generally include a FLAC decoder before API 27. This can be worked around by using * the FLAC extension or the FFmpeg extension. * + *
  • Motion JPEG ({@link MjpegExtractor}) *
  • JPEG ({@link JpegExtractor}) *
  • PNG ({@link PngExtractor}) *
  • WEBP ({@link WebpExtractor}) @@ -124,6 +126,7 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory { // without further analysis. FileTypes.AVI, FileTypes.MIDI, + FileTypes.MJPEG, FileTypes.JPEG, FileTypes.PNG, FileTypes.WEBP, @@ -613,6 +616,9 @@ private void addExtractorsForFileType(@FileTypes.Type int fileType, ListBoth raw concatenated JPEG streams and HTTP {@code multipart/x-mixed-replace} bodies are + * supported. Multipart bodies may declare each frame size with {@code Content-Length}. If they do + * not, the JPEG end-of-image marker is used to find the end of each frame. Optional {@code + * X-Timestamp} part headers are used as presentation timestamps; otherwise frame arrival times are + * used. Raw streams use a default frame rate of 25 frames per second. + */ +@UnstableApi +public final class MjpegExtractor implements Extractor { + + private static final int STATE_DETECTING_STREAM_TYPE = 0; + private static final int STATE_READING_BOUNDARY = 1; + private static final int STATE_READING_HEADERS = 2; + private static final int STATE_SCANNING_SAMPLE = 3; + private static final int STATE_READING_SAMPLE = 4; + + private static final int STREAM_TYPE_UNSET = 0; + private static final int STREAM_TYPE_MULTIPART = 1; + private static final int STREAM_TYPE_RAW = 2; + + private static final int DEFAULT_RAW_FRAME_RATE = 25; + private static final int MAX_HEADER_LINE_LENGTH = 4 * 1024; + private static final int MAX_HEADER_COUNT = 100; + private static final int MAX_SAMPLE_READ_LENGTH = 16 * 1024; + private static final int MAX_SNIFF_BYTES = 4 * 1024 * 1024; + private static final int JPEG_START_OF_IMAGE = 0xFFD8; + private static final int JPEG_END_OF_IMAGE = 0xFFD9; + + private final Clock clock; + private final byte[] scratch; + private final byte[] sampleScanBuffer; + private final StringBuilder lineBuffer; + + private int state; + private int streamType; + private int sampleSize; + private int sampleBytesRemaining; + private int sampleBytesPeeked; + private boolean previousPeekByteWasFF; + private boolean outputFormatSet; + private long firstSampleRealtimeMs; + private long firstPartTimestampUs; + private long partTimestampUs; + private long lastSampleTimeUs; + private long sampleIndex; + private @Nullable String boundary; + private @MonotonicNonNull TrackOutput trackOutput; + + /** Creates an instance. */ + public MjpegExtractor() { + this(Clock.DEFAULT); + } + + @VisibleForTesting + /* package */ MjpegExtractor(Clock clock) { + this.clock = clock; + scratch = new byte[2]; + sampleScanBuffer = new byte[MAX_SAMPLE_READ_LENGTH]; + lineBuffer = new StringBuilder(); + resetState(); + } + + @Override + public boolean sniff(ExtractorInput input) throws IOException { + try { + if (!input.peekFully( + scratch, /* offset= */ 0, /* length= */ 2, /* allowEndOfInput= */ true)) { + return false; + } + int signature = ((scratch[0] & 0xFF) << 8) | (scratch[1] & 0xFF); + input.resetPeekPosition(); + if (signature == JPEG_START_OF_IMAGE) { + return sniffRawStream(input); + } + if (scratch[0] == '-' && scratch[1] == '-') { + return sniffMultipartStream(input); + } + return false; + } catch (EOFException e) { + return false; + } + } + + @Override + public void init(ExtractorOutput output) { + trackOutput = output.track(/* id= */ 0, C.TRACK_TYPE_IMAGE); + output.endTracks(); + output.seekMap(new SeekMap.Unseekable(C.TIME_UNSET)); + } + + @Override + public @ReadResult int read(ExtractorInput input, PositionHolder seekPosition) throws IOException { + switch (state) { + case STATE_DETECTING_STREAM_TYPE: + return detectStreamType(input); + case STATE_READING_BOUNDARY: + return readBoundary(input); + case STATE_READING_HEADERS: + return readHeader(input); + case STATE_SCANNING_SAMPLE: + return scanSample(input); + case STATE_READING_SAMPLE: + return readSample(input); + default: + throw new IllegalStateException(); + } + } + + @Override + public void seek(long position, long timeUs) { + if (position == 0) { + resetState(); + } + } + + @Override + public void release() { + // Do nothing. + } + + private @ReadResult int detectStreamType(ExtractorInput input) throws IOException { + if (!input.peekFully( + scratch, /* offset= */ 0, /* length= */ 2, /* allowEndOfInput= */ true)) { + return RESULT_END_OF_INPUT; + } + input.resetPeekPosition(); + int signature = ((scratch[0] & 0xFF) << 8) | (scratch[1] & 0xFF); + if (signature == JPEG_START_OF_IMAGE) { + streamType = STREAM_TYPE_RAW; + outputFormat(MimeTypes.VIDEO_MJPEG); + startScanningSample(); + } else if (scratch[0] == '-' && scratch[1] == '-') { + streamType = STREAM_TYPE_MULTIPART; + outputFormat(MimeTypes.MULTIPART_MJPEG); + state = STATE_READING_BOUNDARY; + } else { + throw malformedStream("Input is not a raw or multipart MJPEG stream."); + } + return RESULT_CONTINUE; + } + + private @ReadResult int readBoundary(ExtractorInput input) throws IOException { + @Nullable String line; + do { + line = readLine(input); + if (line == null) { + return RESULT_END_OF_INPUT; + } + } while (line.isEmpty()); + + if (boundary == null) { + if (!isBoundary(line)) { + throw malformedStream("Invalid initial MJPEG boundary."); + } + boundary = line; + } else if (line.equals(boundary + "--")) { + return RESULT_END_OF_INPUT; + } else if (!line.equals(boundary)) { + throw malformedStream("Unexpected MJPEG boundary."); + } + sampleSize = C.LENGTH_UNSET; + partTimestampUs = C.TIME_UNSET; + state = STATE_READING_HEADERS; + return RESULT_CONTINUE; + } + + private @ReadResult int readHeader(ExtractorInput input) throws IOException { + @Nullable String line = readLine(input); + if (line == null) { + return RESULT_END_OF_INPUT; + } + if (line.isEmpty()) { + if (sampleSize > 0) { + sampleBytesRemaining = sampleSize; + state = STATE_READING_SAMPLE; + } else { + startScanningSample(); + } + return RESULT_CONTINUE; + } + + int colonIndex = line.indexOf(':'); + if (colonIndex == -1) { + return RESULT_CONTINUE; + } + String headerName = line.substring(0, colonIndex).trim(); + String headerValue = line.substring(colonIndex + 1).trim(); + if (headerName.equalsIgnoreCase("Content-Length")) { + try { + sampleSize = parseContentLength(headerValue); + } catch (NumberFormatException e) { + throw malformedStream("Invalid MJPEG Content-Length header.", e); + } + } else if (headerName.equalsIgnoreCase("X-Timestamp")) { + try { + partTimestampUs = parseTimestampUs(headerValue); + } catch (NumberFormatException e) { + // Ignore malformed optional timestamps and fall back to frame arrival times. + partTimestampUs = C.TIME_UNSET; + } + } + return RESULT_CONTINUE; + } + + private @ReadResult int scanSample(ExtractorInput input) throws IOException { + int bytesPeeked = input.peek(sampleScanBuffer, /* offset= */ 0, sampleScanBuffer.length); + if (bytesPeeked == C.RESULT_END_OF_INPUT) { + input.resetPeekPosition(); + return RESULT_END_OF_INPUT; + } + for (int i = 0; i < bytesPeeked; i++) { + int value = sampleScanBuffer[i] & 0xFF; + if (previousPeekByteWasFF && value == (JPEG_END_OF_IMAGE & 0xFF)) { + sampleSize = sampleBytesPeeked + i + 1; + sampleBytesRemaining = sampleSize; + input.resetPeekPosition(); + state = STATE_READING_SAMPLE; + return RESULT_CONTINUE; + } + previousPeekByteWasFF = value == 0xFF; + } + if (bytesPeeked > Integer.MAX_VALUE - sampleBytesPeeked) { + throw malformedStream("MJPEG frame exceeds the supported sample size."); + } + sampleBytesPeeked += bytesPeeked; + return RESULT_CONTINUE; + } + + private @ReadResult int readSample(ExtractorInput input) throws IOException { + int bytesRead = + checkNotNull(trackOutput) + .sampleData( + input, + Math.min(sampleBytesRemaining, MAX_SAMPLE_READ_LENGTH), + /* allowEndOfInput= */ true); + if (bytesRead == C.RESULT_END_OF_INPUT) { + return RESULT_END_OF_INPUT; + } + sampleBytesRemaining -= bytesRead; + if (sampleBytesRemaining == 0) { + long sampleTimeUs = getSampleTimeUs(); + trackOutput.sampleMetadata( + sampleTimeUs, + BUFFER_FLAG_KEY_FRAME, + sampleSize, + /* offset= */ 0, + /* cryptoData= */ null); + lastSampleTimeUs = sampleTimeUs; + sampleIndex++; + if (streamType == STREAM_TYPE_RAW) { + startScanningSample(); + } else { + state = STATE_READING_BOUNDARY; + } + } + return RESULT_CONTINUE; + } + + private long getSampleTimeUs() { + if (streamType == STREAM_TYPE_RAW) { + return (sampleIndex * C.MICROS_PER_SECOND) / DEFAULT_RAW_FRAME_RATE; + } + long candidateTimeUs; + if (partTimestampUs != C.TIME_UNSET) { + if (firstPartTimestampUs == C.TIME_UNSET) { + firstPartTimestampUs = partTimestampUs; + } + candidateTimeUs = partTimestampUs - firstPartTimestampUs; + } else { + long nowMs = clock.elapsedRealtime(); + if (firstSampleRealtimeMs == C.TIME_UNSET) { + firstSampleRealtimeMs = nowMs; + } + candidateTimeUs = (nowMs - firstSampleRealtimeMs) * 1000; + } + return lastSampleTimeUs == C.TIME_UNSET + ? 0 + : Math.max(lastSampleTimeUs + 1, candidateTimeUs); + } + + private void outputFormat(String containerMimeType) { + if (outputFormatSet) { + return; + } + Format.Builder formatBuilder = + new Format.Builder() + .setContainerMimeType(containerMimeType) + .setSampleMimeType(MimeTypes.IMAGE_JPEG); + if (streamType == STREAM_TYPE_RAW) { + formatBuilder.setFrameRate(DEFAULT_RAW_FRAME_RATE); + } + checkNotNull(trackOutput).format(formatBuilder.build()); + outputFormatSet = true; + } + + private void startScanningSample() { + sampleSize = C.LENGTH_UNSET; + sampleBytesRemaining = 0; + sampleBytesPeeked = 0; + previousPeekByteWasFF = false; + state = STATE_SCANNING_SAMPLE; + } + + private @Nullable String readLine(ExtractorInput input) throws IOException { + while (true) { + int result = input.read(scratch, /* offset= */ 0, /* length= */ 1); + if (result == C.RESULT_END_OF_INPUT) { + if (lineBuffer.length() == 0) { + return null; + } + throw malformedStream("Truncated MJPEG header line."); + } + int value = scratch[0] & 0xFF; + if (value == '\n') { + String line = lineBuffer.toString(); + lineBuffer.setLength(0); + return line; + } + if (value != '\r') { + if (lineBuffer.length() == MAX_HEADER_LINE_LENGTH) { + throw malformedStream("MJPEG header line is too long."); + } + lineBuffer.append((char) value); + } + } + } + + private static boolean sniffMultipartStream(ExtractorInput input) throws IOException { + @Nullable String firstLine = peekLine(input); + if (firstLine == null || !isBoundary(firstLine)) { + return false; + } + for (int i = 0; i < MAX_HEADER_COUNT; i++) { + @Nullable String line = peekLine(input); + if (line == null) { + return false; + } + if (line.isEmpty()) { + byte[] signature = new byte[2]; + input.peekFully(signature, /* offset= */ 0, /* length= */ 2); + return (((signature[0] & 0xFF) << 8) | (signature[1] & 0xFF)) + == JPEG_START_OF_IMAGE; + } + } + return false; + } + + private static boolean sniffRawStream(ExtractorInput input) throws IOException { + byte[] value = new byte[1]; + boolean previousByteWasFF = false; + for (int i = 0; i < MAX_SNIFF_BYTES; i++) { + if (!input.peekFully(value, /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true)) { + return false; + } + int unsignedValue = value[0] & 0xFF; + if (previousByteWasFF && unsignedValue == (JPEG_END_OF_IMAGE & 0xFF)) { + return peekNextJpegStart(input); + } + previousByteWasFF = unsignedValue == 0xFF; + } + return false; + } + + private static boolean peekNextJpegStart(ExtractorInput input) throws IOException { + byte[] value = new byte[1]; + if (!input.peekFully(value, /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true) + || (value[0] & 0xFF) != 0xFF) { + return false; + } + input.peekFully(value, /* offset= */ 0, /* length= */ 1); + return (value[0] & 0xFF) == 0xD8; + } + + private static @Nullable String peekLine(ExtractorInput input) throws IOException { + StringBuilder line = new StringBuilder(); + byte[] value = new byte[1]; + while (true) { + if (!input.peekFully(value, /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true)) { + return line.length() == 0 ? null : line.toString(); + } + int unsignedValue = value[0] & 0xFF; + if (unsignedValue == '\n') { + return line.toString(); + } + if (unsignedValue != '\r') { + if (line.length() == MAX_HEADER_LINE_LENGTH) { + return null; + } + line.append((char) unsignedValue); + } + } + } + + private static boolean isBoundary(String line) { + return line.startsWith("--") && line.length() > 2; + } + + private static int parseContentLength(String value) throws NumberFormatException { + long contentLength = Long.parseLong(value.trim()); + if (contentLength <= 0 || contentLength > Integer.MAX_VALUE) { + throw new NumberFormatException("Content-Length is outside the supported sample size."); + } + return (int) contentLength; + } + + private static long parseTimestampUs(String value) throws NumberFormatException { + int decimalPointIndex = value.indexOf('.'); + String secondsString = decimalPointIndex == -1 ? value : value.substring(0, decimalPointIndex); + String fractionString = decimalPointIndex == -1 ? "" : value.substring(decimalPointIndex + 1); + if (fractionString.length() > 6) { + fractionString = fractionString.substring(0, 6); + } + if (fractionString.length() < 6) { + fractionString = (fractionString + "000000").substring(0, 6); + } + long seconds = Long.parseLong(secondsString); + long fractionUs = fractionString.isEmpty() ? 0 : Long.parseLong(fractionString); + try { + return Math.addExact(Math.multiplyExact(seconds, C.MICROS_PER_SECOND), fractionUs); + } catch (ArithmeticException e) { + throw new NumberFormatException("X-Timestamp is outside the supported range."); + } + } + + private static ParserException malformedStream(String message) { + return malformedStream(message, /* cause= */ null); + } + + private static ParserException malformedStream(String message, @Nullable Throwable cause) { + return ParserException.createForMalformedContainer(message, cause); + } + + private void resetState() { + state = STATE_DETECTING_STREAM_TYPE; + streamType = STREAM_TYPE_UNSET; + sampleSize = C.LENGTH_UNSET; + sampleBytesRemaining = 0; + sampleBytesPeeked = 0; + previousPeekByteWasFF = false; + firstSampleRealtimeMs = C.TIME_UNSET; + firstPartTimestampUs = C.TIME_UNSET; + partTimestampUs = C.TIME_UNSET; + lastSampleTimeUs = C.TIME_UNSET; + sampleIndex = 0; + boundary = null; + lineBuffer.setLength(0); + } +} diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java index aa4bf720aa6..d5bad7bc25e 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java @@ -27,6 +27,7 @@ import androidx.media3.extractor.flv.FlvExtractor; import androidx.media3.extractor.heif.HeifExtractor; import androidx.media3.extractor.jpeg.JpegExtractor; +import androidx.media3.extractor.jpeg.MjpegExtractor; import androidx.media3.extractor.mkv.MatroskaExtractor; import androidx.media3.extractor.mp3.Mp3Extractor; import androidx.media3.extractor.mp4.FragmentedMp4Extractor; @@ -77,6 +78,7 @@ public void createExtractors_withoutMediaInfo_optimizesSniffingOrder() { Ac4Extractor.class, Mp3Extractor.class, AviExtractor.class, + MjpegExtractor.class, JpegExtractor.class, PngExtractor.class, WebpExtractor.class, @@ -101,6 +103,31 @@ public void createExtractors_withMediaInfo_startsWithExtractorsMatchingHeadersAn assertThat(extractorClasses.get(2)).isEqualTo(Mp3Extractor.class); } + @Test + public void createExtractors_withMjpegHeader_startsWithMjpegExtractor() { + DefaultExtractorsFactory defaultExtractorsFactory = new DefaultExtractorsFactory(); + Map> responseHeaders = new HashMap<>(); + responseHeaders.put( + "Content-type", + Collections.singletonList("multipart/x-mixed-replace; boundary=--myboundary")); + + Extractor[] extractors = + defaultExtractorsFactory.createExtractors(Uri.parse("nphMotionJpeg"), responseHeaders); + + assertThat(getUnderlyingExtractorClasses(extractors).get(0)).isEqualTo(MjpegExtractor.class); + } + + @Test + public void createExtractors_withMjpegUri_startsWithMjpegExtractor() { + DefaultExtractorsFactory defaultExtractorsFactory = new DefaultExtractorsFactory(); + + Extractor[] extractors = + defaultExtractorsFactory.createExtractors( + Uri.parse("sample.mjpeg"), /* responseHeaders= */ Collections.emptyMap()); + + assertThat(getUnderlyingExtractorClasses(extractors).get(0)).isEqualTo(MjpegExtractor.class); + } + @Test public void createExtractors_withMediaInfo_optimizesSniffingOrder() { DefaultExtractorsFactory defaultExtractorsFactory = new DefaultExtractorsFactory(); @@ -125,6 +152,7 @@ public void createExtractors_withMediaInfo_optimizesSniffingOrder() { Ac3Extractor.class, Ac4Extractor.class, AviExtractor.class, + MjpegExtractor.class, JpegExtractor.class, PngExtractor.class, WebpExtractor.class, diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/MjpegExtractorTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/MjpegExtractorTest.java new file mode 100644 index 00000000000..88bafb1aeb9 --- /dev/null +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/MjpegExtractorTest.java @@ -0,0 +1,243 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor.jpeg; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.media3.common.C; +import androidx.media3.common.MimeTypes; +import androidx.media3.extractor.Extractor; +import androidx.media3.extractor.PositionHolder; +import androidx.media3.test.utils.FakeClock; +import androidx.media3.test.utils.FakeExtractorInput; +import androidx.media3.test.utils.FakeExtractorOutput; +import androidx.media3.test.utils.FakeTrackOutput; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.primitives.Bytes; +import java.nio.charset.StandardCharsets; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link MjpegExtractor}. */ +@RunWith(AndroidJUnit4.class) +public final class MjpegExtractorTest { + + private static final byte[] FIRST_FRAME = + new byte[] {(byte) 0xFF, (byte) 0xD8, 0x01, (byte) 0xFF, (byte) 0xD9}; + private static final byte[] SECOND_FRAME = + new byte[] {(byte) 0xFF, (byte) 0xD8, 0x02, (byte) 0xFF, (byte) 0xD9}; + + @Test + public void sniff_multipartJpeg_returnsTrue() throws Exception { + MjpegExtractor extractor = new MjpegExtractor(); + + assertThat(extractor.sniff(new FakeExtractorInput.Builder().setData(createStream()).build())) + .isTrue(); + } + + @Test + public void sniff_multipartJpegWithoutContentLength_returnsTrue() throws Exception { + MjpegExtractor extractor = new MjpegExtractor(); + + assertThat( + extractor.sniff( + new FakeExtractorInput.Builder() + .setData(createStreamWithoutContentLength()) + .build())) + .isTrue(); + } + + @Test + public void sniff_doublePrefixedMultipartBoundary_returnsTrue() throws Exception { + MjpegExtractor extractor = new MjpegExtractor(); + byte[] stream = + Bytes.concat( + partHeader("----totalmjpeg", FIRST_FRAME.length), + FIRST_FRAME, + "\r\n".getBytes(StandardCharsets.US_ASCII), + partHeader("----totalmjpeg", SECOND_FRAME.length), + SECOND_FRAME); + + assertThat(extractor.sniff(new FakeExtractorInput.Builder().setData(stream).build())).isTrue(); + } + + @Test + public void sniff_rawMjpeg_returnsTrue() throws Exception { + MjpegExtractor extractor = new MjpegExtractor(); + + assertThat( + extractor.sniff( + new FakeExtractorInput.Builder() + .setData(Bytes.concat(FIRST_FRAME, SECOND_FRAME)) + .build())) + .isTrue(); + } + + @Test + public void sniff_singleJpeg_returnsFalse() throws Exception { + MjpegExtractor extractor = new MjpegExtractor(); + + assertThat(extractor.sniff(new FakeExtractorInput.Builder().setData(FIRST_FRAME).build())) + .isFalse(); + } + + @Test + public void read_multipartJpeg_outputsTimestampedImageSamples() throws Exception { + FakeClock clock = new FakeClock.Builder().setInitialTimeMs(1_000).build(); + MjpegExtractor extractor = new MjpegExtractor(clock); + FakeExtractorOutput output = extractToEnd(extractor, createStream(), clock, 200); + + FakeTrackOutput trackOutput = output.trackOutputs.get(0); + assertThat(trackOutput.getType()).isEqualTo(C.TRACK_TYPE_IMAGE); + assertThat(trackOutput.lastFormat.containerMimeType).isEqualTo(MimeTypes.MULTIPART_MJPEG); + assertThat(trackOutput.lastFormat.sampleMimeType).isEqualTo(MimeTypes.IMAGE_JPEG); + trackOutput.assertSample( + 0, FIRST_FRAME, /* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME, /* cryptoData= */ null); + trackOutput.assertSample( + 1, SECOND_FRAME, /* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME, /* cryptoData= */ null); + assertThat(output.seekMap.isSeekable()).isFalse(); + assertThat(output.seekMap.getDurationUs()).isEqualTo(C.TIME_UNSET); + } + + @Test + public void read_multipartWithoutContentLength_scansJpegMarkers() throws Exception { + FakeClock clock = new FakeClock.Builder().setInitialTimeMs(1_000).build(); + MjpegExtractor extractor = new MjpegExtractor(clock); + + FakeExtractorOutput output = + extractToEnd(extractor, createStreamWithoutContentLength(), clock, 100); + + FakeTrackOutput trackOutput = output.trackOutputs.get(0); + trackOutput.assertSample( + 0, FIRST_FRAME, /* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME, /* cryptoData= */ null); + trackOutput.assertSample( + 1, SECOND_FRAME, /* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME, /* cryptoData= */ null); + } + + @Test + public void read_multipartWithXTimestamp_usesSourceTimestamps() throws Exception { + FakeClock clock = new FakeClock.Builder().setInitialTimeMs(1_000).build(); + MjpegExtractor extractor = new MjpegExtractor(clock); + byte[] stream = + Bytes.concat( + partHeader(FIRST_FRAME.length, "123.100000"), + FIRST_FRAME, + "\r\n".getBytes(StandardCharsets.US_ASCII), + partHeader(SECOND_FRAME.length, "123.350000"), + SECOND_FRAME, + "\r\n--myboundary--\r\n".getBytes(StandardCharsets.US_ASCII)); + + FakeExtractorOutput output = extractToEnd(extractor, stream, clock, /* advanceTimeMs= */ 0); + + FakeTrackOutput trackOutput = output.trackOutputs.get(0); + assertThat(trackOutput.getSampleTimesUs()).containsExactly(0L, 250_000L).inOrder(); + } + + @Test + public void read_rawMjpeg_outputsFramesAtDefaultFrameRate() throws Exception { + FakeClock clock = new FakeClock.Builder().setInitialTimeMs(1_000).build(); + MjpegExtractor extractor = new MjpegExtractor(clock); + + FakeExtractorOutput output = + extractToEnd( + extractor, + Bytes.concat(FIRST_FRAME, SECOND_FRAME), + clock, + /* advanceTimeMs= */ 0); + + FakeTrackOutput trackOutput = output.trackOutputs.get(0); + assertThat(trackOutput.lastFormat.containerMimeType).isEqualTo(MimeTypes.VIDEO_MJPEG); + assertThat(trackOutput.lastFormat.frameRate).isEqualTo(25); + trackOutput.assertSample( + 0, FIRST_FRAME, /* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME, /* cryptoData= */ null); + trackOutput.assertSample( + 1, SECOND_FRAME, /* timeUs= */ 40_000, C.BUFFER_FLAG_KEY_FRAME, /* cryptoData= */ null); + } + + private static byte[] createStream() { + return Bytes.concat( + partHeader(FIRST_FRAME.length), + FIRST_FRAME, + "\r\n".getBytes(StandardCharsets.US_ASCII), + partHeader(SECOND_FRAME.length), + SECOND_FRAME, + "\r\n--myboundary--\r\n".getBytes(StandardCharsets.US_ASCII)); + } + + private static byte[] createStreamWithoutContentLength() { + return Bytes.concat( + partHeaderWithoutContentLength(), + FIRST_FRAME, + "\r\n".getBytes(StandardCharsets.US_ASCII), + partHeaderWithoutContentLength(), + SECOND_FRAME, + "\r\n--myboundary--\r\n".getBytes(StandardCharsets.US_ASCII)); + } + + private static byte[] partHeader(int contentLength) { + return partHeader("--myboundary", contentLength); + } + + private static byte[] partHeader(String boundary, int contentLength) { + return (boundary + + "\r\n" + + "Content-length: " + + contentLength + + "\r\n" + + "Content-type: image/jpeg\r\n" + + "\r\n") + .getBytes(StandardCharsets.US_ASCII); + } + + private static byte[] partHeader(int contentLength, String timestamp) { + return ("--myboundary\r\n" + + "Content-Length: " + + contentLength + + "\r\n" + + "Content-Type: image/jpeg\r\n" + + "X-Timestamp: " + + timestamp + + "\r\n" + + "\r\n") + .getBytes(StandardCharsets.US_ASCII); + } + + private static byte[] partHeaderWithoutContentLength() { + return ("--myboundary\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n") + .getBytes(StandardCharsets.US_ASCII); + } + + private static FakeExtractorOutput extractToEnd( + MjpegExtractor extractor, byte[] stream, FakeClock clock, long advanceTimeMs) + throws Exception { + FakeExtractorInput input = + new FakeExtractorInput.Builder().setData(stream).setSimulatePartialReads(true).build(); + FakeExtractorOutput output = new FakeExtractorOutput(); + extractor.init(output); + PositionHolder positionHolder = new PositionHolder(); + int previousSampleCount = 0; + int result = Extractor.RESULT_CONTINUE; + while (result != Extractor.RESULT_END_OF_INPUT) { + result = extractor.read(input, positionHolder); + FakeTrackOutput trackOutput = output.trackOutputs.get(0); + if (trackOutput.getSampleCount() > previousSampleCount) { + previousSampleCount = trackOutput.getSampleCount(); + clock.advanceTime(advanceTimeMs); + } + } + return output; + } +}