integrate with transformers (#21)

- integrate with transformers (a845831f8f4651b4b6b3676ad744a0a3c368417f)
- Update README (52584706b45157beda5e5f42670d75dfc29bfed7)
- changing custom pipeline and pinning requirements (2fb3b2ec826f02e10b74e334a6a8678273a57dfc)


Co-authored-by: LAin <[email protected]>

MyConfig.py

@@ -0,0 +1,13 @@
+from transformers import PretrainedConfig
+from typing import List
+
+class RMBGConfig(PretrainedConfig):
+    model_type = "SegformerForSemanticSegmentation"
+    def __init__(
+        self,
+        in_ch=3,
+        out_ch=1,
+        **kwargs):
+      self.in_ch = in_ch
+      self.out_ch = out_ch
+      super().__init__(**kwargs)

MyPipe.py

@@ -0,0 +1,73 @@
+import torch, os
+import torch.nn.functional as F
+from torchvision.transforms.functional import normalize
+import numpy as np
+from transformers import Pipeline
+from skimage import io
+from PIL import Image
+
+class RMBGPipe(Pipeline):
+  def __init__(self,**kwargs):
+    Pipeline.__init__(self,**kwargs)
+    self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    self.model.to(self.device)
+    self.model.eval()
+
+  def _sanitize_parameters(self, **kwargs):
+    # parse parameters
+    preprocess_kwargs = {}
+    postprocess_kwargs = {}
+    if "model_input_size" in kwargs : 
+      preprocess_kwargs["model_input_size"] = kwargs["model_input_size"]
+    if "return_mask" in kwargs: 
+      postprocess_kwargs["return_mask"] = kwargs["return_mask"]
+    return preprocess_kwargs, {}, postprocess_kwargs
+
+  def preprocess(self,im_path:str,model_input_size: list=[1024,1024]):
+      # preprocess the input 
+      orig_im = io.imread(im_path)
+      orig_im_size = orig_im.shape[0:2]
+      image = self.preprocess_image(orig_im, model_input_size).to(self.device)
+      inputs = {
+          "image":image,
+          "orig_im_size":orig_im_size,
+          "im_path" : im_path
+      }
+      return inputs
+
+  def _forward(self,inputs):
+    result = self.model(inputs.pop("image"))
+    inputs["result"] = result
+    return inputs
+  def postprocess(self,inputs,return_mask:bool=False ):
+    result = inputs.pop("result")
+    orig_im_size = inputs.pop("orig_im_size")
+    im_path = inputs.pop("im_path")
+    result_image = self.postprocess_image(result[0][0], orig_im_size)
+    pil_im = Image.fromarray(result_image)
+    if return_mask ==True : 
+      return pil_im
+    no_bg_image = Image.new("RGBA", pil_im.size, (0,0,0,0))
+    orig_image = Image.open(im_path)
+    no_bg_image.paste(orig_image, mask=pil_im)
+    return no_bg_image
+    
+  # utilities functions
+  def preprocess_image(self,im: np.ndarray, model_input_size: list=[1024,1024]) -> torch.Tensor:
+    # same as utilities.py with minor modification
+    if len(im.shape) < 3:
+        im = im[:, :, np.newaxis]
+    # orig_im_size=im.shape[0:2]
+    im_tensor = torch.tensor(im, dtype=torch.float32).permute(2,0,1)
+    im_tensor = F.interpolate(torch.unsqueeze(im_tensor,0), size=model_input_size, mode='bilinear').type(torch.uint8)
+    image = torch.divide(im_tensor,255.0)
+    image = normalize(image,[0.5,0.5,0.5],[1.0,1.0,1.0])
+    return image
+  def postprocess_image(self,result: torch.Tensor, im_size: list)-> np.ndarray:
+      result = torch.squeeze(F.interpolate(result, size=im_size, mode='bilinear') ,0)
+      ma = torch.max(result)
+      mi = torch.min(result)
+      result = (result-mi)/(ma-mi)
+      im_array = (result*255).permute(1,2,0).cpu().data.numpy().astype(np.uint8)
+      im_array = np.squeeze(im_array)
+      return im_array

README.md

@@ -2,7 +2,7 @@
 license: other
 license_name: bria-rmbg-1.4
 license_link: https://bria.ai/bria-huggingface-model-license-agreement/
-pipeline_tag: image-to-image
+pipeline_tag: image-segmentation
 tags:
 - remove background
 - background
@@ -10,6 +10,7 @@ tags:
 - Pytorch
 - vision
 - legal liability
+- transformers
 
 extra_gated_prompt: This model weights by BRIA AI can be obtained after a commercial license is agreed upon. Fill in the form below and we reach out to you.
 extra_gated_fields:
@@ -94,43 +95,21 @@ These modifications significantly improve the model’s accuracy and effectivene
 
 ## Installation
 ```bash
-git clone https://huggingface.co/briaai/RMBG-1.4
-cd RMBG-1.4/
-pip install -r requirements.txt
+wget https://huggingface.co/briaai/RMBG-1.4/resolve/main/requirements.txt && pip install -qr requirements.txt
 ```
 
 ## Usage
 
+either load the model 
 ```python
-from skimage import io
-import torch, os
-from PIL import Image
-from briarmbg import BriaRMBG
-from utilities import preprocess_image, postprocess_image
-
-im_path = f"{os.path.dirname(os.path.abspath(__file__))}/example_input.jpg"
-
-net = BriaRMBG.from_pretrained("briaai/RMBG-1.4")
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-net.to(device)
-
-# prepare input
-model_input_size = [1024,1024]
-orig_im = io.imread(im_path)
-orig_im_size = orig_im.shape[0:2]
-image = preprocess_image(orig_im, model_input_size).to(device)
-
-# inference 
-result=net(image)
-
-# post process
-result_image = postprocess_image(result[0][0], orig_im_size)
+from transformers import AutoModelForImageSegmentation
+model = AutoModelForImageSegmentation.from_pretrained("briaai/RMBG-1.4",trust_remote_code=True)
+```
 
-# save result
-pil_im = Image.fromarray(result_image)
-no_bg_image = Image.new("RGBA", pil_im.size, (0,0,0,0))
-orig_image = Image.open(im_path)
-no_bg_image.paste(orig_image, mask=pil_im)
-no_bg_image.save("example_image_no_bg.png")
+or load the pipeline
+```python
+from transformers import pipeline
+pipe = pipeline("image-segmentation", model="briaai/RMBG-1.4", trust_remote_code=True)
+pillow_mask = pipe("img_path",return_mask = True) # outputs a pillow mask
+pillow_image = pipe("image_path") # applies mask on input and returns a pillow image
 ```

briarmbg.py

@@ -1,7 +1,8 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from huggingface_hub import PyTorchModelHubMixin
+from transformers import PreTrainedModel
+from .MyConfig import RMBGConfig
 
 class REBNCONV(nn.Module):
     def __init__(self,in_ch=3,out_ch=3,dirate=1,stride=1):
@@ -345,12 +346,12 @@ class myrebnconv(nn.Module):
         return self.rl(self.bn(self.conv(x)))
 
 
-class BriaRMBG(nn.Module, PyTorchModelHubMixin):
-
-    def __init__(self,config:dict={"in_ch":3,"out_ch":1}):
-        super(BriaRMBG,self).__init__()
-        in_ch=config["in_ch"]
-        out_ch=config["out_ch"]
+class BriaRMBG(PreTrainedModel):
+    config_class = RMBGConfig    
+    def __init__(self,config):
+        super().__init__(config)
+        in_ch = config.in_ch # 3
+        out_ch = config.out_ch # 1
         self.conv_in = nn.Conv2d(in_ch,64,3,stride=2,padding=1)
         self.pool_in = nn.MaxPool2d(2,stride=2,ceil_mode=True)
 

config.json

@@ -1,4 +1,25 @@
 {
-  "in_ch":3,
-  "out_ch":1
-}
+  "_name_or_path": "briaai/RMBG-1.4",
+  "architectures": [
+    "BriaRMBG"
+  ],
+  "auto_map": {
+    "AutoConfig": "MyConfig.RMBGConfig",
+    "AutoModelForImageSegmentation": "briarmbg.BriaRMBG"
+  },
+  "custom_pipelines": {
+    "image-segmentation": {
+      "impl": "MyPipe.RMBGPipe",
+      "pt": [
+        "AutoModelForImageSegmentation"
+      ],
+      "tf": [],
+      "type": "image"
+    }
+  },
+  "in_ch": 3,
+  "model_type": "SegformerForSemanticSegmentation",
+  "out_ch": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.0.dev0"
+}

requirements.txt

@@ -4,4 +4,5 @@ pillow
 numpy
 typing
 scikit-image
-huggingface_hub
+huggingface_hub
+transformers==4.39.1