ui cleanup

app.py

@@ -22,40 +22,15 @@ class face_image_to_face_mesh:
     def __init__(self):
         self.zoe_me = True
         self.uvwrap = not True
-        self.css = ("""
-            #mesh-display-output {
-                max-height: 44vh;
-                max-width:  44vh;
-                width:auto;
-                height:auto
-                }
-            #img-display-output {
-                max-height: 28vh;
-                max-width:  28vh;
-                width:auto;
-                height:auto
-                }
-        """)
 
     def demo(self):
         if self.zoe_me:
             DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
             self.zoe = torch.hub.load('isl-org/ZoeDepth', "ZoeD_N", pretrained=True).to(DEVICE).eval()
 
-        demo = gr.Blocks(css=self.css, cache_examples=True)
+        demo = gr.Blocks(css=self.css(), cache_examples=True)
         with demo:
-            gr.Markdown("""
-                # Face Image to Face Quad Mesh
-
-                Uses MediaPipe to detect a face in an image and convert it to a quad mesh.
-                Saves to OBJ since gltf does not support quad faces.  The 3d viewer has Y pointing the opposite direction from Blender, so ya hafta spin it.
-
-                The face depth with Zoe can be a bit much and without it is a bit generic. In blender you can fix this just by snapping to the high poly model.
-
-                Highly recommend running it locally. The 3D model has uv values in the faces, but you will have to make the mlt file manually at this point."
-
-                Quick import result in examples/converted/movie-gallery.mp4 under files
-            """)
+            gr.Markdown(self.header())
 
             with gr.Row():
                 with gr.Column():
@@ -73,70 +48,14 @@ class face_image_to_face_mesh:
                     upload_image_btn = gr.Button(value="Detect faces")
                     if self.zoe_me:
                         with gr.Group():
-                            use_zoe = gr.Checkbox(label="Use ZoeDepth for Z", value=True)
-                            gr.Textbox(show_label=False, value="Override the MediaPipe depth with ZoeDepth.")
-                            zoe_scale = gr.Slider(label="Zoe Scale", value=1.44, minimum=0.0, maximum=3.3, step=0.1)
-                            gr.Textbox(show_label=False, value="How much to scale the ZoeDepth. 2x is pretty dramatic...")
+                            zoe_scale = gr.Slider(label="Mix the ZoeDepth with the MediaPipe Depth", value=1, minimum=0, maximum=1, step=.01)
+                            flat_scale = gr.Slider(label="Depth scale, smaller is flatter and possibly more flattering", value=1, minimum=0, maximum=1, step=.01)
+                            min_detection_confidence = gr.Slider(label="Mininum face detection confidence", value=.5, minimum=0, maximum=1.0, step=0.01)
                     else:
                         use_zoe = False
                         zoe_scale = 0
                     with gr.Group():
-                        min_detection_confidence = gr.Slider(label="Min detection confidence", value=0.5, minimum=0.0, maximum=1.0, step=0.01)
-                        gr.Textbox(show_label=False, value="Minimum confidence value ([0.0, 1.0]) from the face detection model for the detection to be considered successful.")
-                    with gr.Group():
-                        gr.Markdown(
-                        """
-                        # Using the Textured Mesh in Blender
-
-                        There a couple of annoying steps atm after you download the obj from the 3d viewer. 
-
-                        You can use the script meshin-around.sh in the files section to do the conversion or manually:
-                        
-                        1. edit the file and change the mtllib line to use fun.mtl
-                        2. replace / delete all lines that start with 'f', eg :%s,^f.*,,
-                        3. uncomment all the lines that start with '#f', eg: :%s,^#f,f,
-                        4. save and exit
-                        5. create fun.mtl to point to the texture like:
-
-                        ```
-                        newmtl MyMaterial
-                        map_Kd fun.png
-                        ```
-
-                        Make sure the obj, mtl and png are all in the same directory
-
-                        Now the import will have the texture data: File -> Import -> Wavefront (obj) -> fun.obj
-
-                        This is all a work around for a weird hf+gradios+babylonjs bug which seems to be related to the version
-                        of babylonjs being used... It works fine in a local babylonjs sandbox...
-
-                        # Suggested Workflows
-
-                        Here are some workflow ideas.
-
-                        ## retopologize high poly face mesh
-
-                        1. sculpt high poly mesh in blender
-                        2. snapshot the face
-                        3. generate the mesh using the mediapipe stuff
-                        4. import the low poly mediapipe face
-                        5. snap the mesh to the high poly model
-                        6. model the rest of the low poly model
-                        7. bake the normal / etc maps to the low poly face model
-                        8. it's just that easy 😛
-
-                        Ideally it would be a plugin...
-
-                        ## stable diffusion integration
-
-                        1. generate a face in sd
-                        2. generate the mesh
-                        3. repose it and use it for further generation
-
-                        May need to expanded the generated mesh to cover more, maybe with
-                        <a href="https://github.com/shunsukesaito/PIFu" target="_blank">PIFu model</a>.
-
-                        """)
+                        gr.Markdown(self.footer())
 
                 with gr.Column():
                     with gr.Group():
@@ -147,13 +66,13 @@ class face_image_to_face_mesh:
 
             upload_image_btn.click(
                 fn=self.detect, 
-                inputs=[upload_image, min_detection_confidence,use_zoe,zoe_scale], 
+                inputs=[upload_image, min_detection_confidence,zoe_scale,flat_scale], 
                 outputs=[output_mesh, output_image, depth_image, num_faces_detected]
             )
         demo.launch()
 
 
-    def detect(self, image, min_detection_confidence, use_zoe,zoe_scale):
+    def detect(self, image, min_detection_confidence, zoe_scale, flat_scale):
         width  = image.shape[1]
         height = image.shape[0]
         ratio  = width / height
@@ -164,7 +83,7 @@ class face_image_to_face_mesh:
             
         mesh = "examples/converted/in-granny.obj"
 
-        if self.zoe_me and use_zoe:
+        if self.zoe_me and 0 < zoe_scale:
             depth = self.zoe.infer_pil(image)
             idepth = colorize(depth, cmap='gray_r')
         else:
@@ -182,7 +101,7 @@ class face_image_to_face_mesh:
 
             annotated_image = image.copy()
             for face_landmarks in results.multi_face_landmarks:
-                (mesh,mtl,png) = self.toObj(image=image, width=width, height=height, ratio=ratio, landmark_list=face_landmarks, depth=depth, zoe_scale=zoe_scale)
+                (mesh,mtl,png) = self.toObj(image=image, width=width, height=height, ratio=ratio, landmark_list=face_landmarks, depth=depth, zoe_scale=zoe_scale, flat_scale=flat_scale)
 
                 mp_drawing.draw_landmarks(
                     image=annotated_image,
@@ -201,7 +120,7 @@ class face_image_to_face_mesh:
 
             return mesh, annotated_image, idepth, 1
 
-    def toObj( self, image: np.ndarray, width:int, height:int, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList, depth: np.ndarray, zoe_scale: float):
+    def toObj( self, image: np.ndarray, width:int, height:int, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList, depth: np.ndarray, zoe_scale: float, flat_scale: float):
         print( f'you have such pretty hair', self.temp_dir )
 
         hf_hack = True
@@ -215,7 +134,7 @@ class face_image_to_face_mesh:
             png_file = tempfile.NamedTemporaryFile(suffix='.png', dir=self.temp_dir, delete=False)
 
         ############################################
-        (points,coordinates,colors) = self.landmarksToPoints( image, width, height, ratio, landmark_list, depth, zoe_scale )
+        (points,coordinates,colors) = self.landmarksToPoints( image, width, height, ratio, landmark_list, depth, zoe_scale, flat_scale )
         ############################################
 
         lines = []
@@ -281,7 +200,7 @@ class face_image_to_face_mesh:
         print( f'I know it is special to you so I saved it to {obj_file.name} since we are friends' )
         return (obj_file.name,mtl_file.name,png_file.name)
 
-    def landmarksToPoints( self, image:np.ndarray, width: int, height: int, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList, depth: np.ndarray, zoe_scale: float ):
+    def landmarksToPoints( self, image:np.ndarray, width: int, height: int, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList, depth: np.ndarray, zoe_scale: float, flat_scale: float ):
         points      = [] # 3d vertices
         coordinates = [] # 2d texture coordinates
         colors      = [] # 3d rgb info
@@ -289,21 +208,20 @@ class face_image_to_face_mesh:
         mins = [+np.inf] * 3
         maxs = [-np.inf] * 3
 
-        for idx, landmark in enumerate(landmark_list.landmark):
-            if ((landmark.HasField('visibility') and
-                landmark.visibility < _VISIBILITY_THRESHOLD) or
-                (landmark.HasField('presence') and
-                landmark.presence < _PRESENCE_THRESHOLD)):
-                    idk_what_to_do_for_this = True
+        mp_scale = 1 - zoe_scale
+        print( f'zoe_scale:{zoe_scale}, mp_scale:{mp_scale}' )
 
+        for idx, landmark in enumerate(landmark_list.landmark):
             x, y = _normalized_to_pixel_coordinates(landmark.x,landmark.y,width,height)
             color = image[y,x]
             colors.append( [value / 255 for value in color ] )
             coordinates.append( [x/width,1-y/height] )
 
             if depth is not None:
-                landmark.z = depth[y, x] * zoe_scale
-            #point = [landmark.x * ratio, -landmark.y, -landmark.z];
+                landmark.z = depth[y, x] * zoe_scale + mp_scale * landmark.z
+
+            landmark.z = landmark.z * flat_scale
+
             point = [landmark.x * ratio, landmark.y, landmark.z];
             for pidx,value in enumerate( point ):
                 mins[pidx] = min(mins[pidx],value)
@@ -319,6 +237,7 @@ class face_image_to_face_mesh:
         print( f'maxs: {maxs}' )
         return (points,coordinates,colors)
 
+
     def totallyNormal(self, p0, p1, p2):
         v1 = np.array(p1) - np.array(p0)
         v2 = np.array(p2) - np.array(p0)
@@ -327,6 +246,93 @@ class face_image_to_face_mesh:
         return normal.tolist()    
 
 
+    def header(self):
+        return ("""
+                # Image to Quad Mesh
+
+                Uses MediaPipe to detect a face in an image and convert it to a quad mesh.
+                Saves to OBJ since gltf does not support quad faces.  The 3d viewer has Y pointing the opposite direction from Blender, so ya hafta spin it.
+
+                The face depth with Zoe can be a bit much and without it is a bit generic. In blender you can fix this just by snapping to the high poly model. For photos turning it down to .4 helps, but may still need cleanup...
+
+                Highly recommend running it locally. The 3D model has uv values in the faces, but you will have to either use the script or do some manually tomfoolery.
+
+                Quick import result in examples/converted/movie-gallery.mp4 under files
+        """)
+
+
+    def footer(self):
+        return ( """
+            # Using the Textured Mesh in Blender
+            
+            There a couple of annoying steps atm after you download the obj from the 3d viewer. 
+            
+            You can use the script meshin-around.sh in the files section to do the conversion or manually:
+            
+            1. edit the file and change the mtllib line to use fun.mtl
+            2. replace / delete all lines that start with 'f', eg :%s,^f.*,,
+            3. uncomment all the lines that start with '#f', eg: :%s,^#f,f,
+            4. save and exit
+            5. create fun.mtl to point to the texture like:
+            
+            ```
+            newmtl MyMaterial
+            map_Kd fun.png
+            ```
+            
+            Make sure the obj, mtl and png are all in the same directory
+            
+            Now the import will have the texture data: File -> Import -> Wavefront (obj) -> fun.obj
+            
+            This is all a work around for a weird hf+gradios+babylonjs bug which seems to be related to the version
+            of babylonjs being used... It works fine in a local babylonjs sandbox...
+            
+            # Suggested Workflows
+            
+            Here are some workflow ideas.
+            
+            ## retopologize high poly face mesh
+            
+            1. sculpt high poly mesh in blender
+            2. snapshot the face
+            3. generate the mesh using the mediapipe stuff
+            4. import the low poly mediapipe face
+            5. snap the mesh to the high poly model
+            6. model the rest of the low poly model
+            7. bake the normal / etc maps to the low poly face model
+            8. it's just that easy 😛
+            
+            Ideally it would be a plugin...
+            
+            ## stable diffusion integration
+            
+            1. generate a face in sd
+            2. generate the mesh
+            3. repose it and use it for further generation
+            
+            May need to expanded the generated mesh to cover more, maybe with
+            <a href="https://github.com/shunsukesaito/PIFu" target="_blank">PIFu model</a>.
+            
+        """)
+
+
+    def css(self):
+       return ("""
+            #mesh-display-output {
+                max-height: 44vh;
+                max-width:  44vh;
+                width:auto;
+                height:auto
+                }
+            #img-display-output {
+                max-height: 28vh;
+                max-width:  28vh;
+                width:auto;
+                height:auto
+                }
+        """)
+
+
 face_image_to_face_mesh().demo()
 
 # EOF