yading@10: /* yading@10: * Interface to the Android Stagefright library for yading@10: * H/W accelerated H.264 decoding yading@10: * yading@10: * Copyright (C) 2011 Mohamed Naufal yading@10: * Copyright (C) 2011 Martin Storsjö yading@10: * yading@10: * This file is part of FFmpeg. yading@10: * yading@10: * FFmpeg is free software; you can redistribute it and/or yading@10: * modify it under the terms of the GNU Lesser General Public yading@10: * License as published by the Free Software Foundation; either yading@10: * version 2.1 of the License, or (at your option) any later version. yading@10: * yading@10: * FFmpeg is distributed in the hope that it will be useful, yading@10: * but WITHOUT ANY WARRANTY; without even the implied warranty of yading@10: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU yading@10: * Lesser General Public License for more details. yading@10: * yading@10: * You should have received a copy of the GNU Lesser General Public yading@10: * License along with FFmpeg; if not, write to the Free Software yading@10: * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA yading@10: */ yading@10: yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: #include yading@10: yading@10: extern "C" { yading@10: #include "avcodec.h" yading@10: #include "libavutil/imgutils.h" yading@10: } yading@10: yading@10: #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00 yading@10: yading@10: using namespace android; yading@10: yading@10: struct Frame { yading@10: status_t status; yading@10: size_t size; yading@10: int64_t time; yading@10: int key; yading@10: uint8_t *buffer; yading@10: AVFrame *vframe; yading@10: }; yading@10: yading@10: struct TimeStamp { yading@10: int64_t pts; yading@10: int64_t reordered_opaque; yading@10: }; yading@10: yading@10: class CustomSource; yading@10: yading@10: struct StagefrightContext { yading@10: AVCodecContext *avctx; yading@10: AVBitStreamFilterContext *bsfc; yading@10: uint8_t* orig_extradata; yading@10: int orig_extradata_size; yading@10: sp *source; yading@10: List *in_queue, *out_queue; yading@10: pthread_mutex_t in_mutex, out_mutex; yading@10: pthread_cond_t condition; yading@10: pthread_t decode_thread_id; yading@10: yading@10: Frame *end_frame; yading@10: bool source_done; yading@10: volatile sig_atomic_t thread_started, thread_exited, stop_decode; yading@10: yading@10: AVFrame *prev_frame; yading@10: std::map *ts_map; yading@10: int64_t frame_index; yading@10: yading@10: uint8_t *dummy_buf; yading@10: int dummy_bufsize; yading@10: yading@10: OMXClient *client; yading@10: sp *decoder; yading@10: const char *decoder_component; yading@10: }; yading@10: yading@10: class CustomSource : public MediaSource { yading@10: public: yading@10: CustomSource(AVCodecContext *avctx, sp meta) { yading@10: s = (StagefrightContext*)avctx->priv_data; yading@10: source_meta = meta; yading@10: frame_size = (avctx->width * avctx->height * 3) / 2; yading@10: buf_group.add_buffer(new MediaBuffer(frame_size)); yading@10: } yading@10: yading@10: virtual sp getFormat() { yading@10: return source_meta; yading@10: } yading@10: yading@10: virtual status_t start(MetaData *params) { yading@10: return OK; yading@10: } yading@10: yading@10: virtual status_t stop() { yading@10: return OK; yading@10: } yading@10: yading@10: virtual status_t read(MediaBuffer **buffer, yading@10: const MediaSource::ReadOptions *options) { yading@10: Frame *frame; yading@10: status_t ret; yading@10: yading@10: if (s->thread_exited) yading@10: return ERROR_END_OF_STREAM; yading@10: pthread_mutex_lock(&s->in_mutex); yading@10: yading@10: while (s->in_queue->empty()) yading@10: pthread_cond_wait(&s->condition, &s->in_mutex); yading@10: yading@10: frame = *s->in_queue->begin(); yading@10: ret = frame->status; yading@10: yading@10: if (ret == OK) { yading@10: ret = buf_group.acquire_buffer(buffer); yading@10: if (ret == OK) { yading@10: memcpy((*buffer)->data(), frame->buffer, frame->size); yading@10: (*buffer)->set_range(0, frame->size); yading@10: (*buffer)->meta_data()->clear(); yading@10: (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key); yading@10: (*buffer)->meta_data()->setInt64(kKeyTime, frame->time); yading@10: } else { yading@10: av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n"); yading@10: } yading@10: av_freep(&frame->buffer); yading@10: } yading@10: yading@10: s->in_queue->erase(s->in_queue->begin()); yading@10: pthread_mutex_unlock(&s->in_mutex); yading@10: yading@10: av_freep(&frame); yading@10: return ret; yading@10: } yading@10: yading@10: private: yading@10: MediaBufferGroup buf_group; yading@10: sp source_meta; yading@10: StagefrightContext *s; yading@10: int frame_size; yading@10: }; yading@10: yading@10: void* decode_thread(void *arg) yading@10: { yading@10: AVCodecContext *avctx = (AVCodecContext*)arg; yading@10: StagefrightContext *s = (StagefrightContext*)avctx->priv_data; yading@10: const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(avctx->pix_fmt); yading@10: Frame* frame; yading@10: MediaBuffer *buffer; yading@10: int32_t w, h; yading@10: int decode_done = 0; yading@10: int ret; yading@10: int src_linesize[3]; yading@10: const uint8_t *src_data[3]; yading@10: int64_t out_frame_index = 0; yading@10: yading@10: do { yading@10: buffer = NULL; yading@10: frame = (Frame*)av_mallocz(sizeof(Frame)); yading@10: if (!frame) { yading@10: frame = s->end_frame; yading@10: frame->status = AVERROR(ENOMEM); yading@10: decode_done = 1; yading@10: s->end_frame = NULL; yading@10: goto push_frame; yading@10: } yading@10: frame->status = (*s->decoder)->read(&buffer); yading@10: if (frame->status == OK) { yading@10: sp outFormat = (*s->decoder)->getFormat(); yading@10: outFormat->findInt32(kKeyWidth , &w); yading@10: outFormat->findInt32(kKeyHeight, &h); yading@10: frame->vframe = (AVFrame*)av_mallocz(sizeof(AVFrame)); yading@10: if (!frame->vframe) { yading@10: frame->status = AVERROR(ENOMEM); yading@10: decode_done = 1; yading@10: buffer->release(); yading@10: goto push_frame; yading@10: } yading@10: ret = ff_get_buffer(avctx, frame->vframe); yading@10: if (ret < 0) { yading@10: frame->status = ret; yading@10: decode_done = 1; yading@10: buffer->release(); yading@10: goto push_frame; yading@10: } yading@10: yading@10: // The OMX.SEC decoder doesn't signal the modified width/height yading@10: if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) && yading@10: (w & 15 || h & 15)) { yading@10: if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == buffer->range_length()) { yading@10: w = (w + 15)&~15; yading@10: h = (h + 15)&~15; yading@10: } yading@10: } yading@10: yading@10: if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) { yading@10: avctx->width = w; yading@10: avctx->height = h; yading@10: } yading@10: yading@10: src_linesize[0] = av_image_get_linesize(avctx->pix_fmt, w, 0); yading@10: src_linesize[1] = av_image_get_linesize(avctx->pix_fmt, w, 1); yading@10: src_linesize[2] = av_image_get_linesize(avctx->pix_fmt, w, 2); yading@10: yading@10: src_data[0] = (uint8_t*)buffer->data(); yading@10: src_data[1] = src_data[0] + src_linesize[0] * h; yading@10: src_data[2] = src_data[1] + src_linesize[1] * -(-h>>pix_desc->log2_chroma_h); yading@10: av_image_copy(frame->vframe->data, frame->vframe->linesize, yading@10: src_data, src_linesize, yading@10: avctx->pix_fmt, avctx->width, avctx->height); yading@10: yading@10: buffer->meta_data()->findInt64(kKeyTime, &out_frame_index); yading@10: if (out_frame_index && s->ts_map->count(out_frame_index) > 0) { yading@10: frame->vframe->pts = (*s->ts_map)[out_frame_index].pts; yading@10: frame->vframe->reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque; yading@10: s->ts_map->erase(out_frame_index); yading@10: } yading@10: buffer->release(); yading@10: } else if (frame->status == INFO_FORMAT_CHANGED) { yading@10: if (buffer) yading@10: buffer->release(); yading@10: av_free(frame); yading@10: continue; yading@10: } else { yading@10: decode_done = 1; yading@10: } yading@10: push_frame: yading@10: while (true) { yading@10: pthread_mutex_lock(&s->out_mutex); yading@10: if (s->out_queue->size() >= 10) { yading@10: pthread_mutex_unlock(&s->out_mutex); yading@10: usleep(10000); yading@10: continue; yading@10: } yading@10: break; yading@10: } yading@10: s->out_queue->push_back(frame); yading@10: pthread_mutex_unlock(&s->out_mutex); yading@10: } while (!decode_done && !s->stop_decode); yading@10: yading@10: s->thread_exited = true; yading@10: yading@10: return 0; yading@10: } yading@10: yading@10: static av_cold int Stagefright_init(AVCodecContext *avctx) yading@10: { yading@10: StagefrightContext *s = (StagefrightContext*)avctx->priv_data; yading@10: sp meta, outFormat; yading@10: int32_t colorFormat = 0; yading@10: int ret; yading@10: yading@10: if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1) yading@10: return -1; yading@10: yading@10: s->avctx = avctx; yading@10: s->bsfc = av_bitstream_filter_init("h264_mp4toannexb"); yading@10: if (!s->bsfc) { yading@10: av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n"); yading@10: return -1; yading@10: } yading@10: yading@10: s->orig_extradata_size = avctx->extradata_size; yading@10: s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size + yading@10: FF_INPUT_BUFFER_PADDING_SIZE); yading@10: if (!s->orig_extradata) { yading@10: ret = AVERROR(ENOMEM); yading@10: goto fail; yading@10: } yading@10: memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size); yading@10: yading@10: meta = new MetaData; yading@10: if (meta == NULL) { yading@10: ret = AVERROR(ENOMEM); yading@10: goto fail; yading@10: } yading@10: meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC); yading@10: meta->setInt32(kKeyWidth, avctx->width); yading@10: meta->setInt32(kKeyHeight, avctx->height); yading@10: meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size); yading@10: yading@10: android::ProcessState::self()->startThreadPool(); yading@10: yading@10: s->source = new sp(); yading@10: *s->source = new CustomSource(avctx, meta); yading@10: s->in_queue = new List; yading@10: s->out_queue = new List; yading@10: s->ts_map = new std::map; yading@10: s->client = new OMXClient; yading@10: s->end_frame = (Frame*)av_mallocz(sizeof(Frame)); yading@10: if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client || yading@10: !s->ts_map || !s->end_frame) { yading@10: ret = AVERROR(ENOMEM); yading@10: goto fail; yading@10: } yading@10: yading@10: if (s->client->connect() != OK) { yading@10: av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n"); yading@10: ret = -1; yading@10: goto fail; yading@10: } yading@10: yading@10: s->decoder = new sp(); yading@10: *s->decoder = OMXCodec::Create(s->client->interface(), meta, yading@10: false, *s->source, NULL, yading@10: OMXCodec::kClientNeedsFramebuffer); yading@10: if ((*s->decoder)->start() != OK) { yading@10: av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n"); yading@10: ret = -1; yading@10: s->client->disconnect(); yading@10: goto fail; yading@10: } yading@10: yading@10: outFormat = (*s->decoder)->getFormat(); yading@10: outFormat->findInt32(kKeyColorFormat, &colorFormat); yading@10: if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar || yading@10: colorFormat == OMX_COLOR_FormatYUV420SemiPlanar) yading@10: avctx->pix_fmt = AV_PIX_FMT_NV21; yading@10: else if (colorFormat == OMX_COLOR_FormatYCbYCr) yading@10: avctx->pix_fmt = AV_PIX_FMT_YUYV422; yading@10: else if (colorFormat == OMX_COLOR_FormatCbYCrY) yading@10: avctx->pix_fmt = AV_PIX_FMT_UYVY422; yading@10: else yading@10: avctx->pix_fmt = AV_PIX_FMT_YUV420P; yading@10: yading@10: outFormat->findCString(kKeyDecoderComponent, &s->decoder_component); yading@10: if (s->decoder_component) yading@10: s->decoder_component = av_strdup(s->decoder_component); yading@10: yading@10: pthread_mutex_init(&s->in_mutex, NULL); yading@10: pthread_mutex_init(&s->out_mutex, NULL); yading@10: pthread_cond_init(&s->condition, NULL); yading@10: return 0; yading@10: yading@10: fail: yading@10: av_bitstream_filter_close(s->bsfc); yading@10: av_freep(&s->orig_extradata); yading@10: av_freep(&s->end_frame); yading@10: delete s->in_queue; yading@10: delete s->out_queue; yading@10: delete s->ts_map; yading@10: delete s->client; yading@10: return ret; yading@10: } yading@10: yading@10: static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, yading@10: int *got_frame, AVPacket *avpkt) yading@10: { yading@10: StagefrightContext *s = (StagefrightContext*)avctx->priv_data; yading@10: Frame *frame; yading@10: status_t status; yading@10: int orig_size = avpkt->size; yading@10: AVPacket pkt = *avpkt; yading@10: AVFrame *ret_frame; yading@10: yading@10: if (!s->thread_started) { yading@10: pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx); yading@10: s->thread_started = true; yading@10: } yading@10: yading@10: if (avpkt && avpkt->data) { yading@10: av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size, yading@10: avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY); yading@10: avpkt = &pkt; yading@10: } yading@10: yading@10: if (!s->source_done) { yading@10: if(!s->dummy_buf) { yading@10: s->dummy_buf = (uint8_t*)av_malloc(avpkt->size); yading@10: if (!s->dummy_buf) yading@10: return AVERROR(ENOMEM); yading@10: s->dummy_bufsize = avpkt->size; yading@10: memcpy(s->dummy_buf, avpkt->data, avpkt->size); yading@10: } yading@10: yading@10: frame = (Frame*)av_mallocz(sizeof(Frame)); yading@10: if (avpkt->data) { yading@10: frame->status = OK; yading@10: frame->size = avpkt->size; yading@10: frame->key = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0; yading@10: frame->buffer = (uint8_t*)av_malloc(avpkt->size); yading@10: if (!frame->buffer) { yading@10: av_freep(&frame); yading@10: return AVERROR(ENOMEM); yading@10: } yading@10: uint8_t *ptr = avpkt->data; yading@10: // The OMX.SEC decoder fails without this. yading@10: if (avpkt->size == orig_size + avctx->extradata_size) { yading@10: ptr += avctx->extradata_size; yading@10: frame->size = orig_size; yading@10: } yading@10: memcpy(frame->buffer, ptr, orig_size); yading@10: if (avpkt == &pkt) yading@10: av_free(avpkt->data); yading@10: yading@10: frame->time = ++s->frame_index; yading@10: (*s->ts_map)[s->frame_index].pts = avpkt->pts; yading@10: (*s->ts_map)[s->frame_index].reordered_opaque = avctx->reordered_opaque; yading@10: } else { yading@10: frame->status = ERROR_END_OF_STREAM; yading@10: s->source_done = true; yading@10: } yading@10: yading@10: while (true) { yading@10: if (s->thread_exited) { yading@10: s->source_done = true; yading@10: break; yading@10: } yading@10: pthread_mutex_lock(&s->in_mutex); yading@10: if (s->in_queue->size() >= 10) { yading@10: pthread_mutex_unlock(&s->in_mutex); yading@10: usleep(10000); yading@10: continue; yading@10: } yading@10: s->in_queue->push_back(frame); yading@10: pthread_cond_signal(&s->condition); yading@10: pthread_mutex_unlock(&s->in_mutex); yading@10: break; yading@10: } yading@10: } yading@10: while (true) { yading@10: pthread_mutex_lock(&s->out_mutex); yading@10: if (!s->out_queue->empty()) break; yading@10: pthread_mutex_unlock(&s->out_mutex); yading@10: if (s->source_done) { yading@10: usleep(10000); yading@10: continue; yading@10: } else { yading@10: return orig_size; yading@10: } yading@10: } yading@10: yading@10: frame = *s->out_queue->begin(); yading@10: s->out_queue->erase(s->out_queue->begin()); yading@10: pthread_mutex_unlock(&s->out_mutex); yading@10: yading@10: ret_frame = frame->vframe; yading@10: status = frame->status; yading@10: av_freep(&frame); yading@10: yading@10: if (status == ERROR_END_OF_STREAM) yading@10: return 0; yading@10: if (status != OK) { yading@10: if (status == AVERROR(ENOMEM)) yading@10: return status; yading@10: av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status); yading@10: return -1; yading@10: } yading@10: yading@10: if (s->prev_frame) { yading@10: avctx->release_buffer(avctx, s->prev_frame); yading@10: av_freep(&s->prev_frame); yading@10: } yading@10: s->prev_frame = ret_frame; yading@10: yading@10: *got_frame = 1; yading@10: *(AVFrame*)data = *ret_frame; yading@10: return orig_size; yading@10: } yading@10: yading@10: static av_cold int Stagefright_close(AVCodecContext *avctx) yading@10: { yading@10: StagefrightContext *s = (StagefrightContext*)avctx->priv_data; yading@10: Frame *frame; yading@10: yading@10: if (s->thread_started) { yading@10: if (!s->thread_exited) { yading@10: s->stop_decode = 1; yading@10: yading@10: // Make sure decode_thread() doesn't get stuck yading@10: pthread_mutex_lock(&s->out_mutex); yading@10: while (!s->out_queue->empty()) { yading@10: frame = *s->out_queue->begin(); yading@10: s->out_queue->erase(s->out_queue->begin()); yading@10: if (frame->vframe) { yading@10: avctx->release_buffer(avctx, frame->vframe); yading@10: av_freep(&frame->vframe); yading@10: } yading@10: av_freep(&frame); yading@10: } yading@10: pthread_mutex_unlock(&s->out_mutex); yading@10: yading@10: // Feed a dummy frame prior to signalling EOF. yading@10: // This is required to terminate the decoder(OMX.SEC) yading@10: // when only one frame is read during stream info detection. yading@10: if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) { yading@10: frame->status = OK; yading@10: frame->size = s->dummy_bufsize; yading@10: frame->key = 1; yading@10: frame->buffer = s->dummy_buf; yading@10: pthread_mutex_lock(&s->in_mutex); yading@10: s->in_queue->push_back(frame); yading@10: pthread_cond_signal(&s->condition); yading@10: pthread_mutex_unlock(&s->in_mutex); yading@10: s->dummy_buf = NULL; yading@10: } yading@10: yading@10: pthread_mutex_lock(&s->in_mutex); yading@10: s->end_frame->status = ERROR_END_OF_STREAM; yading@10: s->in_queue->push_back(s->end_frame); yading@10: pthread_cond_signal(&s->condition); yading@10: pthread_mutex_unlock(&s->in_mutex); yading@10: s->end_frame = NULL; yading@10: } yading@10: yading@10: pthread_join(s->decode_thread_id, NULL); yading@10: yading@10: if (s->prev_frame) { yading@10: avctx->release_buffer(avctx, s->prev_frame); yading@10: av_freep(&s->prev_frame); yading@10: } yading@10: yading@10: s->thread_started = false; yading@10: } yading@10: yading@10: while (!s->in_queue->empty()) { yading@10: frame = *s->in_queue->begin(); yading@10: s->in_queue->erase(s->in_queue->begin()); yading@10: if (frame->size) yading@10: av_freep(&frame->buffer); yading@10: av_freep(&frame); yading@10: } yading@10: yading@10: while (!s->out_queue->empty()) { yading@10: frame = *s->out_queue->begin(); yading@10: s->out_queue->erase(s->out_queue->begin()); yading@10: if (frame->vframe) { yading@10: avctx->release_buffer(avctx, frame->vframe); yading@10: av_freep(&frame->vframe); yading@10: } yading@10: av_freep(&frame); yading@10: } yading@10: yading@10: (*s->decoder)->stop(); yading@10: s->client->disconnect(); yading@10: yading@10: if (s->decoder_component) yading@10: av_freep(&s->decoder_component); yading@10: av_freep(&s->dummy_buf); yading@10: av_freep(&s->end_frame); yading@10: yading@10: // Reset the extradata back to the original mp4 format, so that yading@10: // the next invocation (both when decoding and when called from yading@10: // av_find_stream_info) get the original mp4 format extradata. yading@10: av_freep(&avctx->extradata); yading@10: avctx->extradata = s->orig_extradata; yading@10: avctx->extradata_size = s->orig_extradata_size; yading@10: yading@10: delete s->in_queue; yading@10: delete s->out_queue; yading@10: delete s->ts_map; yading@10: delete s->client; yading@10: delete s->decoder; yading@10: delete s->source; yading@10: yading@10: pthread_mutex_destroy(&s->in_mutex); yading@10: pthread_mutex_destroy(&s->out_mutex); yading@10: pthread_cond_destroy(&s->condition); yading@10: av_bitstream_filter_close(s->bsfc); yading@10: return 0; yading@10: } yading@10: yading@10: AVCodec ff_libstagefright_h264_decoder = { yading@10: "libstagefright_h264", yading@10: NULL_IF_CONFIG_SMALL("libstagefright H.264"), yading@10: AVMEDIA_TYPE_VIDEO, yading@10: AV_CODEC_ID_H264, yading@10: CODEC_CAP_DELAY, yading@10: NULL, //supported_framerates yading@10: NULL, //pix_fmts yading@10: NULL, //supported_samplerates yading@10: NULL, //sample_fmts yading@10: NULL, //channel_layouts yading@10: 0, //max_lowres yading@10: NULL, //priv_class yading@10: NULL, //profiles yading@10: sizeof(StagefrightContext), yading@10: NULL, //next yading@10: NULL, //init_thread_copy yading@10: NULL, //update_thread_context yading@10: NULL, //defaults yading@10: NULL, //init_static_data yading@10: Stagefright_init, yading@10: NULL, //encode yading@10: NULL, //encode2 yading@10: Stagefright_decode_frame, yading@10: Stagefright_close, yading@10: };