??

?? ?? ??? ???? ?????? ??? ??? ???? ????. ? ??? ???? ?? ???? ???? ?? ??? ??? ???? ?? ???? ??? ??? ??? ??? ? ????. ????? PC?? ? ?? ????? ??? ?? ??? ?? ? ????.

? ???? Transformers.js? ???? ?? ?? ? ??? ?? ??? ???? ??? ??? ???????. ??? ??? ????

??: https://songlh.top/paint-board/

Github: https://github.com/LHRUN/paint-board Star ??? ?? ?? ?????

Transformers.js

Transformers.js? ??? ??? ???? ?? ?????? ?? ??? ? ?? Hugging Face Transformers? ???? ?? ??? JavaScript ????????. ?? ??? ???? ???? ???? ??? ?? ? ?? ?? ??? ?? ? ??? ?????.

?? Transformers.js? ??? ???? ???? Hugging Face? 1000? ??? ???? ???, ?? ??? ??, ??? ??, ??, ?? ?? ? ?? ?? ?? ? ???? ?? ??? ??? ? ??? Transformers? ?? ?? ??? ? ????. .js ??? ?? ??? ?????.

Transformers.js? ?? ?? ??? V3?? ?????? ?? ??? ??? ?? ??? ???????. Transformers.js v3: WebGPU ??, ? ?? ? ?? ?….

? ???? ??? ? ??? ?? V3??? ??? ? ?? WebGpu ??? ???? ?? ??? ??? ?? ?? ??? ?? ?? ??? ?? ???????. ??, WebGPU? ???? ????? ?? ??? ? ????? ?????? ?? ??? ??? ????? ?? ??????.

?? 1: ?? ??

??? ???? ?? ?? ??? ?? Xenova/modnet ??? ?????

?? ??? ? ??? ?? ? ????

1. ??? ????? ??? ????? ?????.
2. ????? ?????? ?? ???? ?? ?? ???? ?? ???? ????.
3. ??? ?????. ??? ? ???? ?? ??? ??? ???? ???? ???. ???? ??? ???? ?? ?? ?? ??? ???? ???? ?? ???? ???? ?? ? ??? ????.

?? ??? ??? ????. React TS. ??? ??? ? ????? ?? ??? ?????. ?? ??? src/comComponents/boardOperation/uploadImage/index.tsx? ????.

import { useState, FC, useRef, useEffect, useMemo } from 'react'
import {
  env,
  AutoModel,
  AutoProcessor,
  RawImage,
  PreTrainedModel,
  Processor
} from '@huggingface/transformers'

const REMOVE_BACKGROUND_STATUS = {
  LOADING: 0,
  NO_SUPPORT_WEBGPU: 1,
  LOAD_ERROR: 2,
  LOAD_SUCCESS: 3,
  PROCESSING: 4,
  PROCESSING_SUCCESS: 5
}

type RemoveBackgroundStatusType =
  (typeof REMOVE_BACKGROUND_STATUS)[keyof typeof REMOVE_BACKGROUND_STATUS]

const UploadImage: FC<{ url: string }> = ({ url }) => {
  const [removeBackgroundStatus, setRemoveBackgroundStatus] =
    useState<RemoveBackgroundStatusType>()
  const [processedImage, setProcessedImage] = useState('')

  const modelRef = useRef<PreTrainedModel>()
  const processorRef = useRef<Processor>()

  const removeBackgroundBtnTip = useMemo(() => {
    switch (removeBackgroundStatus) {
      case REMOVE_BACKGROUND_STATUS.LOADING:
        return 'Remove background function loading'
      case REMOVE_BACKGROUND_STATUS.NO_SUPPORT_WEBGPU:
        return 'WebGPU is not supported in this browser, to use the remove background function, please use the latest version of Google Chrome'
      case REMOVE_BACKGROUND_STATUS.LOAD_ERROR:
        return 'Remove background function failed to load'
      case REMOVE_BACKGROUND_STATUS.LOAD_SUCCESS:
        return 'Remove background function loaded successfully'
      case REMOVE_BACKGROUND_STATUS.PROCESSING:
        return 'Remove Background Processing'
      case REMOVE_BACKGROUND_STATUS.PROCESSING_SUCCESS:
        return 'Remove Background Processing Success'
      default:
        return ''
    }
  }, [removeBackgroundStatus])

  useEffect(() => {
    ;(async () => {
      try {
        if (removeBackgroundStatus === REMOVE_BACKGROUND_STATUS.LOADING) {
          return
        }
        setRemoveBackgroundStatus(REMOVE_BACKGROUND_STATUS.LOADING)

        // Checking WebGPU Support
        if (!navigator?.gpu) {
          setRemoveBackgroundStatus(REMOVE_BACKGROUND_STATUS.NO_SUPPORT_WEBGPU)
          return
        }
        const model_id = 'Xenova/modnet'
        if (env.backends.onnx.wasm) {
          env.backends.onnx.wasm.proxy = false
        }

        // Load model and processor
        modelRef.current ??= await AutoModel.from_pretrained(model_id, {
          device: 'webgpu'
        })
        processorRef.current ??= await AutoProcessor.from_pretrained(model_id)
        setRemoveBackgroundStatus(REMOVE_BACKGROUND_STATUS.LOAD_SUCCESS)
      } catch (err) {
        console.log('err', err)
        setRemoveBackgroundStatus(REMOVE_BACKGROUND_STATUS.LOAD_ERROR)
      }
    })()
  }, [])

  const processImages = async () => {
    const model = modelRef.current
    const processor = processorRef.current

    if (!model || !processor) {
      return
    }

    setRemoveBackgroundStatus(REMOVE_BACKGROUND_STATUS.PROCESSING)

    // load image
    const img = await RawImage.fromURL(url)

    // Pre-processed image
    const { pixel_values } = await processor(img)

    // Generate image mask
    const { output } = await model({ input: pixel_values })
    const maskData = (
      await RawImage.fromTensor(output[0].mul(255).to('uint8')).resize(
        img.width,
        img.height
      )
    ).data

    // Create a new canvas
    const canvas = document.createElement('canvas')
    canvas.width = img.width
    canvas.height = img.height
    const ctx = canvas.getContext('2d') as CanvasRenderingContext2D

    // Draw the original image
    ctx.drawImage(img.toCanvas(), 0, 0)

    // Updating the mask area
    const pixelData = ctx.getImageData(0, 0, img.width, img.height)
    for (let i = 0; i < maskData.length; ++i) {
      pixelData.data[4 * i + 3] = maskData[i]
    }
    ctx.putImageData(pixelData, 0, 0)

    // Save new image
    setProcessedImage(canvas.toDataURL('image/png'))
    setRemoveBackgroundStatus(REMOVE_BACKGROUND_STATUS.PROCESSING_SUCCESS)
  }

  return (
    <div className="card shadow-xl">
      <button
        className={`btn btn-primary btn-sm ${
          ![
            REMOVE_BACKGROUND_STATUS.LOAD_SUCCESS,
            REMOVE_BACKGROUND_STATUS.PROCESSING_SUCCESS,
            undefined
          ].includes(removeBackgroundStatus)
            ? 'btn-disabled'
            : ''
        }`}
        onClick={processImages}
      >
        Remove background
      </button>
      <div className="text-xs text-base-content mt-2 flex">
        {removeBackgroundBtnTip}
      </div>
      <div className="relative mt-4 border border-base-content border-dashed rounded-lg overflow-hidden">
        <img
          className={`w-[50vw] max-w-[400px] h-[50vh] max-h-[400px] object-contain`}
          src={url}
        />
        {processedImage && (
          <img
            className={`w-full h-full absolute top-0 left-0 z-[2] object-contain`}
            src={processedImage}
          />
        )}
      </div>
    </div>
  )
}

export default UploadImage

?? 2: ??? ?? ??

??? ?? ??? Xenova/slimsam-77-uniform ??? ???? ?????. ??? ??? ????. ???? ??? ? ???? ??? ??? ?? ??? ?????.

?? ??? 5??? ?? ? ????

1. ??? ????? ??? ????? ?????
2. ???? ??? ??? ? ??? ?? ? ?? ???? ???? ?????.
3. ??? ?? ???? ?? ?? ???? ???? ?? ??? ?? ??? ??? ? ? ?? ? ???? ?? ???? ?? ??? ???? ??? ?? ??? ???? ?? ?? ??? ????. .
4. ????? ?????, ??? ? ??? ?? ???? ????? ??? ???? ? ????
5. ???? ???? ???? ??? ?? ???? ?? ?? ??? ???? ???? ?? ??? ???? ?? ?????

?? ??? ??? ????. React TS. ??? ??? ? ????? ?? ??? ?????. ?? ??? src/comComponents/boardOperation/uploadImage/imageSegmentation.tsx? ????.

'react'?? { useState, useRef, useEffect, useMemo, MouseEvent, FC } ????
?? {
  ???,
  ??????,
  ?? ???,
  ?? ??? ??,
  ????,
  ??,
  Sam?????????
} '@huggingface/transformers'??

'@/comComponents/icons/loading.svg?react'?? LoadingIcon ????
'@/comComponents/icons/boardOperation/image-segmentation-????.svg?react'?? PositiveIcon? ?????.
'@/comComponents/icons/boardOperation/image-segmentation-negative.svg?react'?? NegativeIcon? ?????.

????? MarkPoint {
  ??: ??[]
  ??: ??
}

const SEGMENTATION_STATUS = {
  ?? ?: 0,
  NO_SUPPORT_WEBGPU: 1,
  LOAD_ERROR: 2,
  ??_??: 3,
  ?? ?: 4,
  ??_??: 5
}

?? SegmentationStatusType =
  (SEGMENTATION_STATUS ??)[SEGMENTATION_STATUS ?? ?]

const ??? ??: FC<{ url: string }> = ({ URL }) => {
  const [markPoints, setMarkPoints] = useState<MarkPoint[]>([])
  const [segmentationStatus, setSegmentationStatus] =
    useState<SegmentationStatusType>()
  const [pointStatus, setPointStatus] = useState(true)

  const MaskCanvasRef = useRef<HTMLCanvasElement>(null) // ?? ???
  const modelRef = useRef<PreTrainedModel>() // ??
  const processorRef = useRef<processor>() // ????
  const imageInputRef = useRef<RawImage>() // ?? ???
  const imageProcessed = useRef<SamImageProcessorResult>() // ??? ???
  const imageEmbeddings = useRef<tensor>() // ??? ??

  const ??Tip = useMemo(() => {
    ???(????) {
      ??? SEGMENTATION_STATUS.LOADING:
        '??? ?? ?? ?? ?'? ?????.
      ?? SEGMENTATION_STATUS.NO_SUPPORT_WEBGPU:
        return '? ??????? WebGPU? ???? ????. ??? ?? ??? ????? ?? ??? Google Chrome? ?????.'
      ?? SEGMENTATION_STATUS.LOAD_ERROR:
        '??? ?? ??? ???? ?????'? ?????.
      ?? SEGMENTATION_STATUS.LOAD_SUCCESS:
        '??? ?? ??? ????? ???????'? ?????.
      ?? SEGMENTATION_STATUS.PROCESSING:
        '??? ??...'? ?????.
      ?? SEGMENTATION_STATUS.PROCESSING_SUCCESS:
        return '???? ????? ???????. ???? ???? ??? ? ????. ?? ??? ??? ?? ?????.'
      ??:
        ?? ''
    }
  }, [????])

  // 1. ?? ? ???? ??
  useEffect(() => {
    ;(??? () => {
      ???? {
        if (segmentationStatus === SEGMENTATION_STATUS.LOADING) {
          ??
        }

        setSegmentationStatus(SEGMENTATION_STATUS.LOADING)
        if (!navigator?.gpu) {
          setSegmentationStatus(SEGMENTATION_STATUS.NO_SUPPORT_WEBGPU)
          ??
        }const model_id = '???/slimsam-77-uniform'
        modelRef.current ??= SamModel.from_pretrained(model_id, {
          dtype: 'fp16', // ?? "fp32"
          ??: 'webgpu'
        })
        processorRef.current ??= AutoProcessor.from_pretrained(model_id)? ?????.

        setSegmentationStatus(SEGMENTATION_STATUS.LOAD_SUCCESS)
      } ?? (??) {
        console.log('err', ??)
        setSegmentationStatus(SEGMENTATION_STATUS.LOAD_ERROR)
      }
    })()
  }, [])

  // 2. ???? ???
  useEffect(() => {
    ;(??? () => {
      ???? {
        ??? (
          !modelRef.current ||
          !processorRef.current ||
          !url ||
          ?? ?? === SEGMENTATION_STATUS.PROCESSING
        ) {
          ??
        }
        setSegmentationStatus(SEGMENTATION_STATUS.PROCESSING)
        ??????()

        imageInputRef.current = RawImage.fromURL(url)? ?????.
        imageProcessed.current = processorRef.current(? ?????.
          imageInputRef.current
        )
        imageEmbeddings.current = ??(
          modelRef.current? ??? ??
        ).get_image_embeddings(imageProcessed.current)

        setSegmentationStatus(SEGMENTATION_STATUS.PROCESSING_SUCCESS)
      } ?? (??) {
        console.log('err', ??)
      }
    })()
  }, [url, modelRef.current, processorRef.current])

  // ??? ?? ????
  ?? updateMaskOverlay(???: RawImage, ??: Float32Array) {
    const ?????? = ??????Ref.current
    if (!maskCanvas) {
      ??
    }
    const MaskContext = MaskCanvas.getContext('2d') as CanvasRenderingContext2D

    // ??? ?? ????(?? ??)
    if (maskCanvas.width !== ???.? || ??????.?? !== ???.??) {
      ??????.? = ???.?
      ??????.?? = ???.??
    }

    // ?? ???? ?? ?? ??
    const imageData = MaskContext.createImageData(
      ??????.?,
      ??????.??
    )

    // ??? ??? ??
    const numMasks = Score.length // 3
    bestIndex = 0?? ????
    for (let i = 1; i 



<h2>
  
  
  ??
</h2>

<p>????? ?????. ??? ? ?? ?? ?????. ? ?? ????? ??? ??? ???, ???? ????? ?????. ???? ?? ??? ??? ??????!</p>


          

            
        </tensor></processor>

? ??? Canvas ??? ??: Transformers.js? ???? ??? ??? ?? ??? ?? ?????. ??? ??? PHP ??? ????? ?? ?? ??? ?????!