Develop Reference

MediaProjectionManager got freezes after some number of screenshots

I am using MediaProjectionManager for taking screenshots from the ForegroundService. I discovered that the behavior of capturing surface works differently in Android 10 and Android 11.
When I take a screenshot
fun captureBitmap(frame: CropFrames, response: (bitmap: Bitmap) -> Unit){
delayed(50) {
this.frames = frame
projection = mgr!!.getMediaProjection(resultCode, resultData!!)
val cb: MediaProjection.Callback = object : MediaProjection.Callback() {
override fun onStop() {
vdisplay!!.release() //?
response.invoke(latestBitmap!!)
}
}
vdisplay = projection?.createVirtualDisplay(
NAME,
width,
height,
App.densityDpi,
FLAGS,
imageReader.surface,
null,
null
)
projection?.registerCallback(cb, null)
}
}
onImageAvailable triggered
override fun onImageAvailable(reader: ImageReader) {
try {
val image = imageReader.acquireNextImage()
if (image != null) {
val planes = image.planes
val buffer = planes[0].buffer
val pixelStride = planes[0].pixelStride
val rowStride = planes[0].rowStride
val rowPadding = rowStride - pixelStride * width
val bitmapWidth = width + rowPadding / pixelStride
if (latestBitmap == null || latestBitmap!!.width != bitmapWidth || latestBitmap!!.height != height) {
if (latestBitmap != null) {
latestBitmap!!.recycle()
}
latestBitmap = Bitmap.createBitmap(
bitmapWidth,
height, Bitmap.Config.ARGB_8888
)
}
latestBitmap!!.copyPixelsFromBuffer(buffer)
image.close()
handler.parseFrame(frames!!, latestBitmap!!) {
stopCapture()
}
}
} catch (e: Exception) {
e.printStackTrace()
}
}
Then I release it
fun stopCapture() {
if (projection != null) {
projection!!.stop()
vdisplay!!.release()
projection = null
}
}
This flow can be triggered a lot of times per lifecycle, but each call has increased completion time of execution and it looks like short twitches of UI while taking ascreenshot (no main thread calculations). Probably I don't clear something properly? Any suggestions are appreciated. Thanks!

Pixel art creator: loading saved pixel data from Room causes strange Canvas bug/glitch

Note: my question is very specific, apologies if the title isn't clear enough as to what the problem is.
I'm creating a pixel art editor application using Canvas, and the pixel art data is saved into a Room database.
Here's the canvas code:
package com.realtomjoney.pyxlmoose.customviews
import android.content.Context
import android.graphics.*
import android.util.Log
import android.view.MotionEvent
import android.view.View
import androidx.lifecycle.LifecycleOwner
import com.realtomjoney.pyxlmoose.activities.canvas.*
import com.realtomjoney.pyxlmoose.converters.JsonConverter
import com.realtomjoney.pyxlmoose.database.AppData
import com.realtomjoney.pyxlmoose.listeners.CanvasFragmentListener
import com.realtomjoney.pyxlmoose.models.Pixel
import kotlin.math.sqrt
class MyCanvasView(context: Context, val spanCount: Double) : View(context) {
lateinit var extraCanvas: Canvas
lateinit var extraBitmap: Bitmap
val rectangles = mutableMapOf<RectF, Paint?>()
private lateinit var caller: CanvasFragmentListener
private var thisWidth: Int = 0
private var scale: Double = 0.0
override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) {
super.onSizeChanged(w, h, oldw, oldh)
thisWidth = w
caller = context as CanvasFragmentListener
if (::extraBitmap.isInitialized) extraBitmap.recycle()
extraBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
extraCanvas = Canvas(extraBitmap)
scale = (w / spanCount)
for (i in 0 until spanCount.toInt()) {
for (i_2 in 0 until spanCount.toInt()) {
val rect = RectF((i * scale).toFloat(), (i_2 * scale).toFloat(), (i * scale).toFloat() + scale.toFloat(), (i_2 * scale).toFloat() + scale.toFloat())
rectangles[rect] = null
extraCanvas.drawRect(rect, Paint().apply { style = Paint.Style.FILL; color = Color.WHITE })
}
}
}
private fun drawRectAt(x: Float, y: Float) {
for (rect in rectangles.keys) {
if (rect.contains(x, y)) {
caller.onPixelTapped(this, rect)
invalidate()
}
}
}
override fun dispatchTouchEvent(event: MotionEvent): Boolean {
when (event.actionMasked) {
MotionEvent.ACTION_MOVE -> drawRectAt(event.x, event.y)
MotionEvent.ACTION_DOWN -> drawRectAt(event.x, event.y)
}
return true
}
override fun onDraw(canvas: Canvas) {
super.onDraw(canvas)
canvas.drawBitmap(extraBitmap, 0f, 0f, null)
}
fun saveData(): List<Pixel> {
val data = mutableListOf<Pixel>()
for (pair in rectangles) {
data.add(Pixel(pair.value?.color))
}
return data
}
fun loadData(context: LifecycleOwner, index: Int) {
AppData.db.pixelArtCreationsDao().getAllPixelArtCreations().observe(context, {
currentPixelArtObj = it[index]
val localPixelData = JsonConverter.convertJsonStringToPixelList(currentPixelArtObj.pixelData)
var index = 0
for (i in 0 until sqrt(localPixelData.size.toDouble()).toInt()) {
for (i_2 in 0 until sqrt(localPixelData.size.toDouble()).toInt()) {
val rect = RectF((i * scale).toFloat(), (i_2 * scale).toFloat(), (i * scale).toFloat() + scale.toFloat(), (i_2 * scale).toFloat() + scale.toFloat())
rectangles[rect] = null
extraCanvas.drawRect(rect, Paint().apply { style = Paint.Style.FILL; isAntiAlias = false; color = localPixelData[index].pixelColor ?: Color.WHITE })
rectangles[rectangles.keys.toList()[index]] = Paint().apply { style = Paint.Style.FILL; isAntiAlias = false; color = localPixelData[index].pixelColor ?: Color.WHITE }
index++
}
}
})
}
}
Here's an example of how a 10 by 10 canvas may look like:
The pixel data is saved into a Room database as a Json String, and whenever we want to access this data we convert the Json String back to a List<Pixel>, et cetera:
Dao:
#Dao
interface PixelArtCreationsDao {
#Insert
suspend fun insertPixelArt(pixelArt: PixelArt)
#Query("SELECT * FROM PixelArt ")
fun getAllPixelArtCreations(): LiveData<List<PixelArt>>
#Query("DELETE FROM PixelArt WHERE objId=:pixelArtId")
fun deletePixelArtCreation(pixelArtId: Int)
#Query("UPDATE PixelArt SET item_bitmap=:bitmap WHERE objId=:id_t")
fun updatePixelArtCreationBitmap(bitmap: String, id_t: Int): Int
#Query("UPDATE PixelArt SET item_pixel_data=:pixelData WHERE objId=:id_t")
fun updatePixelArtCreationPixelData(pixelData: String, id_t: Int): Int
#Query("UPDATE PixelArt SET item_favourited=:favorited WHERE objId=:id_t")
fun updatePixelArtCreationFavorited(favorited: Boolean, id_t: Int): Int
}
PixelArt database:
#Database(entities = [PixelArt::class], version = 1)
abstract class PixelArtDatabase: RoomDatabase() {
abstract fun pixelArtCreationsDao(): PixelArtCreationsDao
companion object {
private var instance: PixelArtDatabase? = null
fun getDatabase(context: Context): PixelArtDatabase {
if (instance == null) {
synchronized(PixelArtDatabase::class) {
if (instance == null) instance = Room.databaseBuilder(context.applicationContext, PixelArtDatabase::class.java, AppData.dbFileName).allowMainThreadQueries().build()
}
}
return instance!!
}
}
}
AppData:
class AppData {
companion object {
var dbFileName = "pixel_art_db"
lateinit var db: PixelArtDatabase
}
}
Model:
#Entity
data class PixelArt(
#ColumnInfo(name = "item_bitmap") var bitmap: String,
#ColumnInfo(name = "item_title") var title: String,
#ColumnInfo(name = "item_pixel_data") var pixelData: String,
#ColumnInfo(name = "item_favourited") var favourited: Boolean,
#ColumnInfo(name = "item_date_created") var dateCreated: String = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").format(LocalDateTime.now())) {
#PrimaryKey(autoGenerate = true) var objId = 0
}
Now, say we have two projects like so with two different spanCount values:
Once we click on the first item, the following occurs:
For some reason it's setting the grid size to be equal to that of the second item, and I'm really trying to understand why this is the case. I've tried for a couple of hours to fix this weird glitch and have had no luck doing so.
But, for some reason when we go to our second item it renders properly:
If we create a new 80 x 80 canvas and then go back to the second creation it will render like so:
I'm assuming that it's setting the spanCount to that of the latest item in the database, but I'm unsure why this is happening.
I suspect it has something to do with the code that takes the List<Pixel> and draws it onscreen:
fun loadData(context: LifecycleOwner, index: Int) {
AppData.db.pixelArtCreationsDao().getAllPixelArtCreations().observe(context, {
currentPixelArtObj = it[index]
val localPixelData = JsonConverter.convertJsonStringToPixelList(currentPixelArtObj.pixelData)
var index = 0
for (i in 0 until sqrt(localPixelData.size.toDouble()).toInt()) {
for (i_2 in 0 until sqrt(localPixelData.size.toDouble()).toInt()) {
val rect = RectF((i * scale).toFloat(), (i_2 * scale).toFloat(), (i * scale).toFloat() + scale.toFloat(), (i_2 * scale).toFloat() + scale.toFloat())
rectangles[rect] = null
extraCanvas.drawRect(rect, Paint().apply { style = Paint.Style.FILL; isAntiAlias = false; color = localPixelData[index].pixelColor ?: Color.WHITE })
rectangles[rectangles.keys.toList()[index]] = Paint().apply { style = Paint.Style.FILL; isAntiAlias = false; color = localPixelData[index].pixelColor ?: Color.WHITE }
index++
}
}
})
}
Although I'm not entirely sure where the source of the bug is coming from because it seems I'm doing everything right. It's honestly been a brainfuck trying to fix this lol
Any help would be appreciated to fix this annoying glitch so I can finish my pixel art editor app.

This bug was fixed by calling invalidate() on the Fragment's Canvas property after the user taps the back button. It took me a couple of days to get to fix this, so I'm posting an answer here in case someone has a similar bug.
fun CanvasActivity.extendedOnBackPressed() {
canvasFragmentInstance.myCanvasViewInstance.invalidate()
startActivity(Intent(context, MainActivity::class.java))
}

Camera X captures images in different rotation states

Okay, I went through different posts and find out that depending on mobile manufacturers there can be a complications such as capture images get rotated, so you have to be aware of that. What I did was:
fun rotateBitmap(bitmap: Bitmap): Bitmap? {
val matrix = Matrix()
when (getImageOrientation(bitmap)) {
ExifInterface.ORIENTATION_NORMAL -> return bitmap
ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.setScale(-1f, 1f)
ExifInterface.ORIENTATION_ROTATE_270 -> matrix.setRotate(-90f)
ExifInterface.ORIENTATION_ROTATE_180 -> matrix.setRotate(180f)
ExifInterface.ORIENTATION_ROTATE_90 -> matrix.setRotate(90f)
ExifInterface.ORIENTATION_FLIP_VERTICAL -> {
matrix.setRotate(180f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSPOSE -> {
matrix.setRotate(90f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSVERSE -> {
matrix.setRotate(-90f)
matrix.postScale(-1f, 1f)
}
else -> return bitmap
}
This worked. But then I noticed something really weird and that might be related with how I configured Camera X configuration.
With the same device I get differently rotated Bitmaps (well, this should not happen. If devices rotates image weirdly, it should rotate images in both modes - in ImageAnalysesUseCase and ImageCaptureUseCase).
So, why is this happening and how can I fix it?
Code implementation:
Binding camera X to life-cycle:
CameraX.bindToLifecycle(
this,
buildPreviewUseCase(),
buildImageAnalysisUseCase(),
buildImageCaptureUseCase()
)
Preview use case:
private fun buildPreviewUseCase(): Preview {
val previewConfig = PreviewConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetResolution(config.resolution)
.setTargetRotation(Surface.ROTATION_0)
.setLensFacing(config.lensFacing)
.build()
return AutoFitPreviewBuilder.build(previewConfig, cameraTextureView)
}
Capture use case:
private fun buildImageCaptureUseCase(): ImageCapture {
val captureConfig = ImageCaptureConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetRotation(Surface.ROTATION_0)
.setTargetResolution(config.resolution)
.setCaptureMode(config.captureMode)
.build()
val capture = ImageCapture(captureConfig)
manualModeTakePhotoButton.setOnClickListener {
capture.takePicture(object : ImageCapture.OnImageCapturedListener() {
override fun onCaptureSuccess(imageProxy: ImageProxy, rotationDegrees: Int) {
viewModel.onManualCameraModeAnalysis(imageProxy, rotationDegrees)
}
override fun onError(useCaseError: ImageCapture.UseCaseError?, message: String?, cause: Throwable?) {
//
}
})
}
return capture
}
Analysis use case:
private fun buildImageAnalysisUseCase(): ImageAnalysis {
val analysisConfig = ImageAnalysisConfig.Builder().apply {
val analyzerThread = HandlerThread("xAnalyzer").apply { start() }
analyzerHandler = Handler(analyzerThread.looper)
setCallbackHandler(analyzerHandler!!)
setTargetAspectRatio(config.aspectRatio)
setTargetRotation(Surface.ROTATION_0)
setTargetResolution(config.resolution)
setImageReaderMode(config.readerMode)
setImageQueueDepth(config.queueDepth)
}.build()
val analysis = ImageAnalysis(analysisConfig)
analysis.analyzer = ImageRecognitionAnalyzer(viewModel)
return analysis
}
AutoFitPreviewBuilder:
class AutoFitPreviewBuilder private constructor(config: PreviewConfig,
viewFinderRef: WeakReference<TextureView>) {
/** Public instance of preview use-case which can be used by consumers of this adapter */
val useCase: Preview
/** Internal variable used to keep track of the use-case's output rotation */
private var bufferRotation: Int = 0
/** Internal variable used to keep track of the view's rotation */
private var viewFinderRotation: Int? = null
/** Internal variable used to keep track of the use-case's output dimension */
private var bufferDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's dimension */
private var viewFinderDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's display */
private var viewFinderDisplay: Int = -1
/** Internal reference of the [DisplayManager] */
private lateinit var displayManager: DisplayManager
/**
* We need a display listener for orientation changes that do not trigger a configuration
* change, for example if we choose to override config change in manifest or for 180-degree
* orientation changes.
*/
private val displayListener = object : DisplayManager.DisplayListener {
override fun onDisplayAdded(displayId: Int) = Unit
override fun onDisplayRemoved(displayId: Int) = Unit
override fun onDisplayChanged(displayId: Int) {
val viewFinder = viewFinderRef.get() ?: return
if (displayId == viewFinderDisplay) {
val display = displayManager.getDisplay(displayId)
val rotation = getDisplaySurfaceRotation(display)
updateTransform(viewFinder, rotation, bufferDimens, viewFinderDimens)
}
}
}
init {
// Make sure that the view finder reference is valid
val viewFinder = viewFinderRef.get() ?:
throw IllegalArgumentException("Invalid reference to view finder used")
// Initialize the display and rotation from texture view information
viewFinderDisplay = viewFinder.display.displayId
viewFinderRotation = getDisplaySurfaceRotation(viewFinder.display) ?: 0
// Initialize public use-case with the given config
useCase = Preview(config)
// Every time the view finder is updated, recompute layout
useCase.onPreviewOutputUpdateListener = Preview.OnPreviewOutputUpdateListener {
val viewFinder =
viewFinderRef.get() ?: return#OnPreviewOutputUpdateListener
// To update the SurfaceTexture, we have to remove it and re-add it
val parent = viewFinder.parent as ViewGroup
parent.removeView(viewFinder)
parent.addView(viewFinder, 0)
viewFinder.surfaceTexture = it.surfaceTexture
bufferRotation = it.rotationDegrees
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, it.textureSize, viewFinderDimens)
}
// Every time the provided texture view changes, recompute layout
viewFinder.addOnLayoutChangeListener { view, left, top, right, bottom, _, _, _, _ ->
val viewFinder = view as TextureView
val newViewFinderDimens = Size(right - left, bottom - top)
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, bufferDimens, newViewFinderDimens)
}
// Every time the orientation of device changes, recompute layout
displayManager = viewFinder.context
.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
displayManager.registerDisplayListener(displayListener, null)
// Remove the display listeners when the view is detached to avoid
// holding a reference to the View outside of a Fragment.
// NOTE: Even though using a weak reference should take care of this,
// we still try to avoid unnecessary calls to the listener this way.
viewFinder.addOnAttachStateChangeListener(object : View.OnAttachStateChangeListener {
override fun onViewAttachedToWindow(view: View?) {
displayManager.registerDisplayListener(displayListener, null)
}
override fun onViewDetachedFromWindow(view: View?) {
displayManager.unregisterDisplayListener(displayListener)
}
})
}
/** Helper function that fits a camera preview into the given [TextureView] */
private fun updateTransform(textureView: TextureView?, rotation: Int?, newBufferDimens: Size,
newViewFinderDimens: Size) {
// This should not happen anyway, but now the linter knows
val textureView = textureView ?: return
if (rotation == viewFinderRotation &&
Objects.equals(newBufferDimens, bufferDimens) &&
Objects.equals(newViewFinderDimens, viewFinderDimens)) {
// Nothing has changed, no need to transform output again
return
}
if (rotation == null) {
// Invalid rotation - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderRotation = rotation
}
if (newBufferDimens.width == 0 || newBufferDimens.height == 0) {
// Invalid buffer dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
bufferDimens = newBufferDimens
}
if (newViewFinderDimens.width == 0 || newViewFinderDimens.height == 0) {
// Invalid view finder dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderDimens = newViewFinderDimens
}
val matrix = Matrix()
// Compute the center of the view finder
val centerX = viewFinderDimens.width / 2f
val centerY = viewFinderDimens.height / 2f
// Correct preview output to account for display rotation
matrix.postRotate(-viewFinderRotation!!.toFloat(), centerX, centerY)
// Buffers are rotated relative to the device's 'natural' orientation: swap width and height
val bufferRatio = bufferDimens.height / bufferDimens.width.toFloat()
val scaledWidth: Int
val scaledHeight: Int
// Match longest sides together -- i.e. apply center-crop transformation
if (viewFinderDimens.width > viewFinderDimens.height) {
scaledHeight = viewFinderDimens.width
scaledWidth = Math.round(viewFinderDimens.width * bufferRatio)
} else {
scaledHeight = viewFinderDimens.height
scaledWidth = Math.round(viewFinderDimens.height * bufferRatio)
}
// Compute the relative scale value
val xScale = scaledWidth / viewFinderDimens.width.toFloat()
val yScale = scaledHeight / viewFinderDimens.height.toFloat()
// Scale input buffers to fill the view finder
matrix.preScale(xScale, yScale, centerX, centerY)
// Finally, apply transformations to our TextureView
textureView.setTransform(matrix)
}
companion object {
/** Helper function that gets the rotation of a [Display] in degrees */
fun getDisplaySurfaceRotation(display: Display?) = when(display?.rotation) {
Surface.ROTATION_0 -> 0
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> null
}
/**
* Main entrypoint for users of this class: instantiates the adapter and returns an instance
* of [Preview] which automatically adjusts in size and rotation to compensate for
* config changes.
*/
fun build(config: PreviewConfig, viewFinder: TextureView) =
AutoFitPreviewBuilder(config, WeakReference(viewFinder)).useCase
}
}
If configuration is correct (it looks okay to me), then next idea was that maybe converting captured images objects to bitmap might be faulty. Below you can see implementation.
Capture mode uses this function:
fun imageProxyToBitmap(image: ImageProxy): Bitmap {
val buffer: ByteBuffer = image.planes[0].buffer
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
Analysis mode uses this function:
fun toBitmapFromImage(image: Image?): Bitmap? {
try {
if (image == null || image.planes[0] == null || image.planes[1] == null || image.planes[2] == null) {
return null
}
val yBuffer = image.planes[0].buffer
val uBuffer = image.planes[1].buffer
val vBuffer = image.planes[2].buffer
val ySize = yBuffer.remaining()
val uSize = uBuffer.remaining()
val vSize = vBuffer.remaining()
val nv21 = ByteArray(ySize + uSize + vSize)
/* U and V are swapped */
yBuffer.get(nv21, 0, ySize)
vBuffer.get(nv21, ySize, vSize)
uBuffer.get(nv21, ySize + vSize, uSize)
val yuvImage = YuvImage(nv21, ImageFormat.NV21, image.width, image.height, null)
val out = ByteArrayOutputStream()
yuvImage.compressToJpeg(Rect(0, 0, yuvImage.width, yuvImage.height), 50, out)
val imageBytes = out.toByteArray()
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
} catch (e: IllegalStateException) {
Log.e("IllegalStateException", "#ImageUtils.toBitmapFromImage(): Can't read the image file.")
return null
}
}
So, weirdly, on few devices toBitmapFromImage() sometimes comes up upwards, but at the same time (same device) imageProxyToBitmap() returns image in correct rotation - it has to be the image to bitmap functions fault, right?Why is this happening (because capture mode returns image normally) and how to fix this?

Inside onImageCaptureSuccess, get the rotationDegrees and rotate your bitmap by that degree to get the correct orientation.
override fun onImageCaptureSuccess(image: ImageProxy) {
val capturedImageBitmap = image.image?.toBitmap()?.rotate(image.imageInfo.rotationDegrees.toFloat())
mBinding.previewImage.setImageBitmap(capturedImageBitmap)
showPostClickViews()
mCurrentFlow = FLOW_CAMERA
}
toBitmap() and rotate() are extension functions.
fun Image.toBitmap(): Bitmap {
val buffer = planes[0].buffer
buffer.rewind()
val bytes = ByteArray(buffer.capacity())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
fun Bitmap.rotate(degrees: Float): Bitmap =
Bitmap.createBitmap(this, 0, 0, width, height, Matrix().apply { postRotate(degrees) }, true)

CameraX returns the captured image with a rotation value in the callback, which can be used to rotate the image.
https://developer.android.com/reference/androidx/camera/core/ImageCapture.OnImageCapturedListener.html#onCaptureSuccess(androidx.camera.core.ImageProxy,%20int)

For Analyzer UseCases, you have to get rotationDegree coming through analyze method of ImageAnalysis.Analyzer and work accordingly.
Hope it helps!

ARCore with Twilio how is it implemented?

I would like to implement ARCore with Twilio's video call. The documentation says this is possible but I could not figure out how to do it. Can Anyone tell me what I'm doing wrong?
This is my activity:
class MixActivity : AppCompatActivity() {
private lateinit var mArFragment: ArFragment
private lateinit var mVideoView: ArSceneView
private var mScreenVideoTrack: LocalVideoTrack? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_mix)
mArFragment = ar_fragment as ArFragment
mVideoView = mArFragment.arSceneView
mScreenVideoTrack = LocalVideoTrack.create(this, true,
ViewCapturer(mVideoView)
)
} }
This is view:
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="#+id/container"
android:layout_width="match_parent"
android:layout_height="match_parent">
<fragment
android:id="#+id/ar_fragment"
android:name="com.google.ar.sceneform.ux.ArFragment"
android:layout_width="match_parent"
android:layout_height="match_parent"/>
</RelativeLayout>
And VideoCapture:
internal class ViewCapturer(private val view: View) : VideoCapturer, PixelCopy.OnPixelCopyFinishedListener {
private val handler = Handler(Looper.getMainLooper())
private var videoCapturerListener: VideoCapturer.Listener? = null
private val started = AtomicBoolean(false)
private lateinit var mViewBitmap: Bitmap
private val viewCapturer = object : Runnable {
override fun run() {
val dropFrame = view.width == 0 || view.height == 0
// Only capture the view if the dimensions have been established
if (!dropFrame) {
// Draw view into bitmap backed canvas
val measuredWidth = View.MeasureSpec.makeMeasureSpec(
view.width,
View.MeasureSpec.EXACTLY
)
val measuredHeight = View.MeasureSpec.makeMeasureSpec(
view.height,
View.MeasureSpec.EXACTLY
)
view.measure(measuredWidth, measuredHeight)
view.layout(0, 0, view.measuredWidth, view.measuredHeight)
mViewBitmap = Bitmap.createBitmap(
view.width, view.height,
Bitmap.Config.ARGB_8888
)
val viewCanvas = Canvas(mViewBitmap)
view.draw(viewCanvas)
// Extract the frame from the bitmap
val bytes = mViewBitmap.byteCount
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener!!.onFrameCaptured(videoFrame)
}
}
// Schedule the next capture
if (started.get()) {
handler.postDelayed(this, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
}
/**
* Returns the list of supported formats for this view capturer. Currently, only supports
* capturing to RGBA_8888 bitmaps.
*
* #return list of supported formats.
*/
override fun getSupportedFormats(): List<VideoFormat> {
val videoFormats = ArrayList<VideoFormat>()
val videoDimensions = VideoDimensions(view.width, view.height)
val videoFormat = VideoFormat(videoDimensions, 30, VideoPixelFormat.RGBA_8888)
videoFormats.add(videoFormat)
return videoFormats
}
/**
* Returns true because we are capturing screen content.
*/
override fun isScreencast(): Boolean {
return true
}
/**
* This will be invoked when it is time to start capturing frames.
*
* #param videoFormat the video format of the frames to be captured.
* #param listener capturer listener.
*/
override fun startCapture(videoFormat: VideoFormat, listener: VideoCapturer.Listener) {
// Store the capturer listener
this.videoCapturerListener = listener
this.started.set(true)
// Notify capturer API that the capturer has started
val capturerStarted = handler.postDelayed(
viewCapturer,
VIEW_CAPTURER_FRAMERATE_MS.toLong()
)
this.videoCapturerListener!!.onCapturerStarted(capturerStarted)
}
/**
* Stop capturing frames. Note that the SDK cannot receive frames once this has been invoked.
*/
override fun stopCapture() {
this.started.set(false)
handler.removeCallbacks(viewCapturer)
}
override fun onPixelCopyFinished(i: Int) {
// Extract the frame from the bitmap
val bytes = mViewBitmap.getByteCount()
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener?.onFrameCaptured(videoFrame)
}
if (started.get()) {
handler.postDelayed(viewCapturer, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
companion object {
private val VIEW_CAPTURER_FRAMERATE_MS = 100
}
}
The ARCore part works but the Twilio part does not work.
I referred to another post that talked about it but it was incomplete:
Streaming CustomView ARcore with Twilio video

Rotating images in Listview

I am trying to make a fake horizontal ListView. For that, I want to rotate the content of each cell in the adapter. It almost works:
class ImageAdapter(ctx: Context, viewResourceId: Int, imageResourceId: Int, strings: Array[String], icons: TypedArray, colorSelected: Int, colorNotSelected: Int, colorBorder: Int) extends ArrayAdapter[String](ctx, viewResourceId, strings) {
// Useful variables
private val mInflater: LayoutInflater = ctx.getSystemService(
Context.LAYOUT_INFLATER_SERVICE).asInstanceOf[LayoutInflater]
private val mStrings: Array[String] = strings
private val mIcons: TypedArray = icons
private val mViewResourceId: Int = viewResourceId
private var mSelected : Int = 0
def setSelected(s: Int) = mSelected = s
// Inherited from ArrayAdapter
override def getCount():Int = mStrings.length
override def getItem(position: Int): String = mStrings(position)
override def getItemId(position: Int): Long = position
// The rotation machinery
var rotation = 90
private var matrix = new Matrix()
val rect1 = new RectF(0, 0, 0, 0)
val rect2 = new RectF(0, 0, 0, 0)
val cst = Matrix.ScaleToFit.CENTER
// Getting the view.
override def getView(position: Int, convertView: View, parent: ViewGroup): View = {
var result = if(convertView == null) mInflater.inflate(mViewResourceId, null) else convertView
var iv: ImageView = result.findViewById(imageResourceId).asInstanceOf[ImageView]
val drawable = mIcons.getDrawable(position)
iv.setScaleType(ScaleType.MATRIX)
matrix.reset()
val height = drawable.getIntrinsicHeight()
val width = drawable.getIntrinsicWidth()
rotation match {
case 0 =>
rect1.set(0, 0, width, height)
rect2.set(0, 0, iv.getWidth(), iv.getHeight())
matrix.setRectToRect(rect1, rect2, cst)
case 90 =>
rect1.set(0, 0, width, height)
rect2.set(-iv.getHeight()/2, -iv.getWidth()/2, iv.getHeight()/2, iv.getWidth()/2)
matrix.setRectToRect(rect1, rect2, cst)
matrix.postRotate(rotation, 0, 0)
matrix.postTranslate(iv.getWidth()/2, iv.getHeight()/2)
...
case _ =>
}
iv.setImageMatrix(matrix)
iv.setImageDrawable(drawable)
iv.postInvalidate()
if(position == mSelected) {
result.setBackgroundColor(colorSelected)
} else {
result.setBackgroundColor(colorNotSelected)
}
result.setTag(mStrings(position))
result
}
}
The only problem is that it does not display the first picture. Instead, it displays this:
For the first list to appear, I need to click on one element of each list. The onclick method is the following:
listGraphismesView.setOnItemClickListener(new OnItemClickListener {
override def onItemClick(parent: AdapterView[_], view: View, position: Int, notUsedId: Long) = {
var graphismeName = view.getTag().asInstanceOf[String]
mAdapter.setSelected(position)
mAdapter.notifyDataSetChanged()
showGChallenge(position, graphismeName)
}
})
I tried then to put mAdapter.notifyDataSetChanged() in the onResume function of the fragment, but it does not update the list from the beginning. I still need to click on something for all images to appear.
Any idea why ?

I found out that iv.getWidth() and iv.getHeight() were both returning zero because the getview would be called by the onActivityCreated function. Putting notifyDataSetChanged() in the onResume function was too early too. So the drawable would be drawn with size 0, and reloaded when I could click on the button.
So to change that, I wrote the following piece of code, calling the resize function with a listener.
var iv: ImageView = result.findViewById(imageResourceId).asInstanceOf[ImageView]
val drawable = mIcons.getDrawable(position)
iv.setImageDrawable(drawable)
iv.setScaleType(ScaleType.MATRIX)
val vto = iv.getViewTreeObserver()
vto.addOnGlobalLayoutListener(new OnGlobalLayoutListener() {
private var matrix = new Matrix()
val rect1 = new RectF(0, 0, 0, 0)
val rect2 = new RectF(0, 0, 0, 0)
private var height = drawable.getIntrinsicHeight()
private var width = drawable.getIntrinsicWidth()
override def onGlobalLayout() {
val iv_width = iv.getWidth
val iv_height = iv.getHeight
matrix.reset()
rotation match {
case 0 =>
rect1.set(0, 0, width, height)
rect2.set(0, 0, iv_width, iv_height)
matrix.setRectToRect(rect1, rect2, cst)
case 90 =>
rect1.set(0, 0, width, height)
rect2.set(-iv_height/2, -iv_width/2, iv_height/2, iv_width/2)
matrix.setRectToRect(rect1, rect2, cst)
matrix.postRotate(rotation, 0, 0)
matrix.postTranslate(iv_width/2, iv_height/2)
...
case _ =>
}
iv.setImageMatrix(matrix)
}
})