Upload 82 files

.gitattributes

@@ -5,3 +5,12 @@ RobustVideoMatting/documentation/image/showreel.gif filter=lfs diff=lfs merge=lf
 RobustVideoMatting/documentation/image/teaser.gif filter=lfs diff=lfs merge=lfs -text
 RobustVideoMatting/rvm_mobilenetv3.pth filter=lfs diff=lfs merge=lfs -text
 pifuhd/checkpoints/pifuhd.pt filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smpl/models/basicmodel_f_lbs_10_207_0_v1.1.0.pkl filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smpl/models/basicmodel_m_lbs_10_207_0_v1.0.0.pkl filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smpl/models/basicmodel_m_lbs_10_207_0_v1.1.0.pkl filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smpl/models/basicmodel_neutral_lbs_10_207_0_v1.1.0.pkl filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smplify/models/basicModel_neutral_lbs_10_207_0_v1.0.0.pkl filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smplify/models/regressors_locked_normalized_female.npz filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smplify/models/regressors_locked_normalized_hybrid.npz filter=lfs diff=lfs merge=lfs -text
+videoavatars/vendor/smplify/models/regressors_locked_normalized_male.npz filter=lfs diff=lfs merge=lfs -text

videoavatars/.github/ISSUE_TEMPLATE/new-issue.md

@@ -0,0 +1,13 @@
+---
+name: New issue
+about: Default issue template.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+- [ ] I have read the paper **Video based reconstruction of 3D people models**.
+- [ ] I have read the license.
+- [ ] I understand that the authors are under no obligation to reply to comprehension questions regarding the paper or the provided code.
+- [ ] The issue occurs in the provided code and not in third-party libraries (SMPL, chumpy, OpenDR, ...)

videoavatars/.gitignore

@@ -0,0 +1,6 @@
+.idea/
+.DS_Store
+
+*.pyc
+vendor/*
+!vendor/__init__.py

videoavatars/README.md

@@ -0,0 +1,89 @@
+# Video Avatars
+
+This repository contains code corresponding to the paper **Video based reconstruction of 3D people models**.
+
+## Installation
+
+Download and unpack the SMPL model from here: http://smpl.is.tue.mpg.de/.
+
+Download and unpack the SMPLify Code from here: http://smplify.is.tue.mpg.de/
+
+
+```
+git clone https://github.com/thmoa/videoavatars.git videoavatars
+cd videoavatars/vendor
+mkdir smpl
+cd smpl
+ln -s <path to smpl folder>/models .
+ln -s <path to smpl folder>/smpl_webuser/*.py .
+cd ..
+
+mkdir smplify
+cd smplify
+touch __init__.py
+ln -s <path to your smplify installation>/code/models .
+# this file needs to be copied!
+cp <path to your smplify installation>/code/lib/sphere_collisions.py .
+# these files can be linked
+ln -s <path to smplify folder>/code/lib/capsule_body.py .
+ln -s <path to smplify folder>/code/lib/capsule_ch.py .
+ln -s <path to smplify folder>/code/lib/robustifiers.py .
+```
+
+Change line 14 in `vendor/smplify/sphere_collisions.py` to
+```
+from vendor.smpl.lbs import verts_core
+```
+
+## Usage
+
+The software consists of three parts:
+
+1. `step1_pose.py`: pose reconstruction
+2. `step2_consensus.py`: consensus shape optimization
+3. `step3_texture.py`: texture calculation
+
+Starting the scripts will display usage information and options.
+Additionally, we provide helper bash scripts for easier processing of our dataset.
+
+The scripts in `prepare_data` might help you to process your own data.
+
+## Citation
+
+This repository contains code corresponding to:
+
+T. Alldieck, M. Magnor, W. Xu, C. Theobalt, and G. Pons-
+Moll. **Video based reconstruction of 3D people models**. In
+*IEEE Conference on Computer Vision and Pattern Recognition*, 2018.
+
+Please cite as:
+
+```
+@inproceedings{alldieck2018video,
+  title = {Video Based Reconstruction of 3D People Models},
+  author = {Alldieck, Thiemo and Magnor, Marcus and Xu, Weipeng and Theobalt, Christian and Pons-Moll, Gerard},
+  booktitle = {{IEEE} Conference on Computer Vision and Pattern Recognition},
+  year = {2018}
+}
+```
+
+
+## Dataset
+
+The accompanied dataset can be downloaded here: https://graphics.tu-bs.de/people-snapshot
+
+## License
+
+Copyright (c) 2018 Thiemo Alldieck, Technische Universität Braunschweig, Max-Planck-Gesellschaft
+
+**Please read carefully the following terms and conditions and any accompanying documentation before you download and/or use this software and associated documentation files (the "Software").**
+
+The authors hereby grant you a non-exclusive, non-transferable, free of charge right to copy, modify, merge, publish, distribute, and sublicense the Software for the sole purpose of performing non-commercial scientific research, non-commercial education, or non-commercial artistic projects.
+
+Any other use, in particular any use for commercial purposes, is prohibited. This includes, without limitation, incorporation in a commercial product, use in a commercial service, or production of other artefacts for commercial purposes.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+You understand and agree that the authors are under no obligation to provide either maintenance services, update services, notices of latent defects, or corrections of defects with regard to the Software. The authors nevertheless reserve the right to update, modify, or discontinue the Software at any time.
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. You agree to cite the **Video Based Reconstruction of 3D People Models** paper in documents and papers that report on research using this Software.

videoavatars/get_reconstructed_poses.py

@@ -0,0 +1,56 @@
+'''
+@Author: Jiapeng Zhou
+@Date: 2023-05-26 15:55:30
+@Description: This is to get the smpl params "reconstructed_poses.hdf5" from videoavatar.
+Usage: conda activate videoavatar && pythonn get_reconstructed_poses.py --root ~/Deexxx/xinyu_a --out videoavatars --gender xxx
+'''
+import argparse
+import os, os.path as osp, sys
+import pdb
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', type=str)
+    parser.add_argument('--out', type=str, default='videoavatars')
+    parser.add_argument('--gender', type=str, choices=['male', 'female'],help=' "male" or "female" run_step1.sh needs to select smpl model of different gender')
+    args = parser.parse_args()
+    return args
+
+def main():
+    args = parse_args() 
+    output_dir = osp.join(args.root, args.out)
+    #os.makedirs(output_dir, exist_ok=True)
+    # openpose => keypoints.hdf5
+    output_pth =osp.join(output_dir, 'keypoints.hdf5') 
+    if not osp.isfile(output_pth):
+        os.system('python ./prepare_data/2djoints2hdf5.py {} {}'.\
+                 format(osp.join(args.root, 'openpose'), output_pth) )
+
+    # masks => masks.hdf5
+    output_pth = osp.join(output_dir, 'masks.hdf5')
+    if not osp.isfile(output_pth):
+        os.system('python ./prepare_data/masks2hdf5.py {} {}'.\
+                 format(osp.join(args.root, 'masks'), output_pth))
+
+    # camera => camera.hdf5
+    # TODO: the f and c is given by the camera intrinsics
+    output_pth = osp.join(output_dir, 'camera.pkl')
+    '''the compute formula:
+    fx2 = fy, fy2 = fx, cx2 = 1080 - cy, cy2 = cx
+    '''
+    if not osp.isfile(output_pth):
+#        os.system('python ./prepare_data/create_camera.py \
+#                  {} 1080 1920 -f 914.87 915.067 -c 540.63 958.13 '\
+#                  .format(output_pth))
+        os.system('python ./prepare_data/create_camera.py \
+                  {} 1080 1920  '\
+                  .format(output_pth))
+
+    # get reconstructed_poses.hdf5
+    # TODO: the smpl model is needs to be selected according to gender
+    os.system('bash run_step1.sh {} {} {}'.format(output_dir, output_dir, args.gender))
+    return
+
+if __name__ == '__main__':
+    main()

videoavatars/get_reconstructed_poses.sh

@@ -0,0 +1,4 @@
+# 如果用python调用这个bash脚本 os.system('bash xxx.sh')，会直接报错commandNotFoundError
+conda activate videoavatar
+python get_reconstructed_poses.py  --root $1 --out $2 --gender $3
+caself

videoavatars/lib/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+

videoavatars/lib/ch.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import chumpy as ch
+import scipy.sparse as sp
+
+from chumpy.utils import col
+
+
+class sp_dot(ch.Ch):
+    terms = 'a',
+    dterms = 'b',
+
+    def compute_r(self):
+        return self.a.dot(self.b.r)
+
+    def compute(self):
+
+        # To stay consistent with numpy, we must upgrade 1D arrays to 2D
+        ar = sp.csr_matrix((self.a.data, self.a.indices, self.a.indptr),
+                            shape=(max(np.sum(self.a.shape[:-1]), 1), self.a.shape[-1]))
+        br = col(self.b.r) if len(self.b.r.shape) < 2 else self.b.r.reshape((self.b.r.shape[0], -1))
+
+        if br.ndim <= 1:
+            return ar
+        elif br.ndim <= 2:
+            return sp.kron(ar, sp.eye(br.shape[1], br.shape[1]))
+        else:
+            raise NotImplementedError
+
+    def compute_dr_wrt(self, wrt):
+        if wrt is self.b:
+            return self.compute()

videoavatars/lib/frame.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import numpy as np
+
+from util.logger import log
+from lib.rays import rays_from_silh
+from models.smpl import model_params_in_camera_coords
+
+sess = None
+
+
+class FrameData(object):
+    pass
+
+
+def batch_invert(x):
+    try:
+        import tensorflow as tf
+        global sess
+
+        tx = tf.convert_to_tensor(x, dtype=tf.float32)
+        txi = tf.transpose(tf.matrix_inverse(tf.transpose(tx)))
+
+        if sess is None:
+            sess = tf.Session()
+
+        return sess.run(txi)
+
+    except ImportError:
+        log.info('Could not load tensorflow. Falling back to matrix inversion with numpy (slower).')
+
+    return np.asarray([np.linalg.inv(t) for t in x.T]).T
+
+
+def setup_frame_rays(base_smpl, camera, camera_t, camera_rt, pose, trans, mask):
+    f = FrameData()
+
+    f.trans, f.pose = model_params_in_camera_coords(trans, pose, base_smpl.J[0], camera_t, camera_rt)
+    f.mask = mask
+
+    base_smpl.pose[:] = f.pose
+    camera.t[:] = f.trans
+
+    f.Vi = batch_invert(base_smpl.V.r)
+    f.rays = rays_from_silh(f.mask, camera)
+
+    return f

videoavatars/lib/geometry.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import cv2
+import numpy as np
+from scipy import sparse as sp
+
+
+def visible_boundary_edges(rn_b, rn_m):
+    visibility = rn_b.boundaryid_image
+
+    silh = rn_m.r
+    sobelx = cv2.Sobel(silh, cv2.CV_64F, 1, 0, ksize=3)
+    sobely = cv2.Sobel(silh, cv2.CV_64F, 0, 1, ksize=3)
+
+    mag = (sobelx ** 2 + sobely ** 2) > 0
+
+    visibility[mag == 0] = 4294967295
+    visible = np.nonzero(visibility.ravel() != 4294967295)[0]
+
+    return np.unique(visibility.ravel()[visible])
+
+
+def visible_boundary_edge_verts(rn_b, rn_m):
+    visible_edge_ids = visible_boundary_edges(rn_b, rn_m)
+
+    vpe = rn_b.primitives_per_edge[1]
+    verts = np.unique(vpe[visible_edge_ids].ravel())
+
+    return verts
+
+
+def laplacian(v, f):
+    n = len(v)
+
+    v_a = f[:, 0]
+    v_b = f[:, 1]
+    v_c = f[:, 2]
+
+    ab = v[v_a] - v[v_b]
+    bc = v[v_b] - v[v_c]
+    ca = v[v_c] - v[v_a]
+
+    cot_a = -1 * (ab * ca).sum(axis=1) / np.sqrt(np.sum(np.cross(ab, ca) ** 2, axis=-1))
+    cot_b = -1 * (bc * ab).sum(axis=1) / np.sqrt(np.sum(np.cross(bc, ab) ** 2, axis=-1))
+    cot_c = -1 * (ca * bc).sum(axis=1) / np.sqrt(np.sum(np.cross(ca, bc) ** 2, axis=-1))
+
+    I = np.concatenate((v_a, v_c, v_a, v_b, v_b, v_c))
+    J = np.concatenate((v_c, v_a, v_b, v_a, v_c, v_b))
+    W = 0.5 * np.concatenate((cot_b, cot_b, cot_c, cot_c, cot_a, cot_a))
+
+    L = sp.csr_matrix((W, (I, J)), shape=(n, n))
+    L = L - sp.spdiags(L * np.ones(n), 0, n, n)
+
+    return L

videoavatars/lib/rays.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import cv2
+import numpy as np
+import chumpy as ch
+
+from lib.geometry import visible_boundary_edge_verts
+from vendor.smplify.robustifiers import GMOf
+
+
+def plucker(rays):
+    p = rays[:, 0]
+    n = rays[:, 1] - rays[:, 0]
+    n /= np.linalg.norm(n, axis=1).reshape(-1, 1)
+    m = np.cross(p, n, axisa=1, axisb=1)
+
+    return n, m
+
+
+def distance_function(rays, verts):
+    n, m = plucker(rays)
+    return ch.cross(verts, n, axisa=1, axisb=1) - m
+
+
+def unpose_and_select_rays(rays, Vi, smpl, rn_b, rn_m):
+    v_ids = visible_boundary_edge_verts(rn_b, rn_m)
+    verts = smpl.r[v_ids]
+
+    n, m = plucker(rays)
+    dist = np.linalg.norm(np.cross(verts.reshape(-1, 1, 3), n, axisa=2, axisb=1) - m, axis=2)
+
+    ray_matches = np.argmin(dist, axis=0)
+    vert_matches = np.argmin(dist, axis=1)
+
+    rays_u_r = np.zeros_like(rays)
+
+    M = Vi[:, :, v_ids]
+    T = smpl.v_posevariation[v_ids].r
+
+    tmp0 = M[:, :, ray_matches] * np.hstack((rays[:, 0], np.ones((rays.shape[0], 1)))).T.reshape(1, 4, -1)
+    tmp1 = M[:, :, ray_matches] * np.hstack((rays[:, 1], np.ones((rays.shape[0], 1)))).T.reshape(1, 4, -1)
+
+    rays_u_r[:, 0] = np.sum(tmp0, axis=1).T[:, :3] - T[ray_matches]
+    rays_u_r[:, 1] = np.sum(tmp1, axis=1).T[:, :3] - T[ray_matches]
+
+    rays_u_v = np.zeros_like(rays[vert_matches])
+
+    tmp0 = M * np.hstack((rays[vert_matches, 0], np.ones((verts.shape[0], 1)))).T.reshape(1, 4, -1)
+    tmp1 = M * np.hstack((rays[vert_matches, 1], np.ones((verts.shape[0], 1)))).T.reshape(1, 4, -1)
+
+    rays_u_v[:, 0] = np.sum(tmp0, axis=1).T[:, :3] - T
+    rays_u_v[:, 1] = np.sum(tmp1, axis=1).T[:, :3] - T
+
+    valid_rays = dist[np.vstack((ray_matches, range(dist.shape[1]))).tolist()] < 0.12
+    valid_verts = dist[np.vstack((range(dist.shape[0]), vert_matches)).tolist()] < 0.03
+
+    ray_matches = ray_matches[valid_rays]
+
+    return np.concatenate((v_ids[ray_matches], v_ids[valid_verts])), \
+           np.concatenate((rays_u_r[valid_rays], rays_u_v[valid_verts]))
+
+
+def rays_from_points(points, camera):
+    points = np.hstack((points, np.ones((points.shape[0], 1))))
+    points3d = camera.unproject_points(points)
+
+    c0 = -camera.t.r
+
+    return np.hstack((np.repeat(c0.reshape(1, 1, 3), points3d.shape[0], axis=0), points3d.reshape(-1, 1, 3)))
+
+
+def rays_from_silh(mask, camera):
+
+    if cv2.__version__[0] == '2':
+        contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+    else:
+        _, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+    silh = np.zeros_like(mask)
+
+    for con in contours:
+        cv2.drawContours(silh, [con], 0, 1, 1)
+
+    points = np.vstack(np.where(silh == 1)[::-1]).astype(np.float32).T
+    rays = rays_from_points(points, camera)
+
+    return rays
+
+
+def ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m):
+    base_smpl.pose[:] = f.pose
+    camera.t[:] = f.trans
+
+    f.v_ids, f.rays_u = unpose_and_select_rays(f.rays, f.Vi, base_smpl, vis_rn_b, vis_rn_m)
+    f.verts = base_smpl.v_shaped_personal[f.v_ids]
+    f.dist = distance_function(f.rays_u, f.verts)
+
+    return GMOf(f.dist, sigma)

videoavatars/models/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-

videoavatars/models/bodyparts.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import cPickle as pkl
+import numpy as np
+
+_cache = None
+
+
+def get_bodypart_vertex_ids():
+    global _cache
+
+    if _cache is None:
+        with open('assets/bodyparts.pkl', 'rb') as fp:
+            _cache = pkl.load(fp)
+
+    return _cache
+
+
+def faces_no_hands(f):
+    v_ids = get_bodypart_vertex_ids()
+    hands = np.concatenate((v_ids['hand_r'], v_ids['hand_l']))
+    return np.array(filter(lambda face: np.intersect1d(face, hands).size == 0, f))
+
+
+def regularize_laplace():
+    reg = np.ones(6890)
+    v_ids = get_bodypart_vertex_ids()
+
+    reg[v_ids['face']] = 12.
+    reg[v_ids['hand_l']] = 5.
+    reg[v_ids['hand_r']] = 5.
+    reg[v_ids['fingers_l']] = 8.
+    reg[v_ids['fingers_r']] = 8.
+    reg[v_ids['foot_l']] = 5.
+    reg[v_ids['foot_r']] = 5.
+    reg[v_ids['toes_l']] = 8.
+    reg[v_ids['toes_r']] = 8.
+    reg[v_ids['ear_l']] = 10.
+    reg[v_ids['ear_r']] = 10.
+
+    return reg
+
+
+def regularize_model():
+    reg = np.ones(6890)
+    v_ids = get_bodypart_vertex_ids()
+
+    reg[v_ids['face']] = 7.
+    reg[v_ids['hand_l']] = 12.
+    reg[v_ids['hand_r']] = 12.
+    reg[v_ids['fingers_l']] = 15.
+    reg[v_ids['fingers_r']] = 15.
+    reg[v_ids['foot_l']] = 12.
+    reg[v_ids['foot_r']] = 12.
+    reg[v_ids['toes_l']] = 15.
+    reg[v_ids['toes_r']] = 15.
+    reg[v_ids['ear_l']] = 10.
+    reg[v_ids['ear_r']] = 10.
+
+    return reg
+
+
+def regularize_symmetry():
+    reg = np.ones(6890)
+    v_ids = get_bodypart_vertex_ids()
+
+    reg[v_ids['face']] = 10.
+    reg[v_ids['hand_l']] = 10.
+    reg[v_ids['hand_r']] = 10.
+    reg[v_ids['foot_l']] = 10.
+    reg[v_ids['foot_r']] = 10.
+    reg[v_ids['ear_l']] = 5.
+    reg[v_ids['ear_r']] = 5.
+
+    return reg

videoavatars/models/smpl.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import chumpy as ch
+import numpy as np
+import cPickle as pkl
+import scipy.sparse as sp
+from chumpy.ch import Ch
+from vendor.smpl.posemapper import posemap, Rodrigues
+from vendor.smpl.serialization import backwards_compatibility_replacements
+
+
+VERT_NOSE = 331
+VERT_EAR_L = 3485
+VERT_EAR_R = 6880
+VERT_EYE_L = 2802
+VERT_EYE_R = 6262
+
+
+class Smpl(Ch):
+    """
+    Class to store SMPL object with slightly improved code and access to more matrices
+    """
+    terms = 'model',
+    dterms = 'trans', 'betas', 'pose', 'v_personal'
+
+    def __init__(self, *args, **kwargs):
+        self.on_changed(self._dirty_vars)
+
+    def on_changed(self, which):
+        if not hasattr(self, 'trans'):
+            self.trans = ch.zeros(3)
+
+        if not hasattr(self, 'betas'):
+            self.betas = ch.zeros(10)
+
+        if not hasattr(self, 'pose'):
+            self.pose = ch.zeros(72)
+
+        if 'model' in which:
+            if not isinstance(self.model, dict):
+                dd = pkl.load(open(self.model))
+            else:
+                dd = self.model
+
+            backwards_compatibility_replacements(dd)
+
+            for s in ['posedirs', 'shapedirs']:
+                if (s in dd) and not hasattr(dd[s], 'dterms'):
+                    dd[s] = ch.array(dd[s])
+
+            self.f = dd['f']
+            self.v_template = dd['v_template']
+            if not hasattr(self, 'v_personal'):
+                self.v_personal = ch.zeros_like(self.v_template)
+            self.shapedirs = dd['shapedirs']
+            self.J_regressor = dd['J_regressor']
+            if 'J_regressor_prior' in dd:
+                self.J_regressor_prior = dd['J_regressor_prior']
+            if sp.issparse(self.J_regressor):
+                self.J_regressor = self.J_regressor.toarray()
+            self.bs_type = dd['bs_type']
+            self.weights = dd['weights']
+            if 'vert_sym_idxs' in dd:
+                self.vert_sym_idxs = dd['vert_sym_idxs']
+            if 'weights_prior' in dd:
+                self.weights_prior = dd['weights_prior']
+            self.kintree_table = dd['kintree_table']
+            self.posedirs = dd['posedirs']
+
+            self._set_up()
+
+    def _set_up(self):
+        self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template
+        self.v_shaped_personal = self.v_shaped + self.v_personal
+        self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) * self.v_shaped.reshape(-1, 3, 1), axis=0).T
+        self.v_posevariation = self.posedirs.dot(posemap(self.bs_type)(self.pose))
+        self.v_poseshaped = self.v_shaped_personal + self.v_posevariation
+
+        self.A, A_global = self._global_rigid_transformation()
+        self.Jtr = ch.vstack([g[:3, 3] for g in A_global])
+        self.J_transformed = self.Jtr + self.trans.reshape((1, 3))
+
+        self.V = self.A.dot(self.weights.T)
+
+        rest_shape_h = ch.hstack((self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1))))
+        self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3]
+        self.v = self.v_posed + self.trans
+
+    def _global_rigid_transformation(self):
+        results = {}
+        pose = self.pose.reshape((-1, 3))
+        parent = {i: self.kintree_table[0, i] for i in range(1, self.kintree_table.shape[1])}
+
+        with_zeros = lambda x: ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
+        pack = lambda x: ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))])
+
+        results[0] = with_zeros(ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1)))))
+
+        for i in range(1, self.kintree_table.shape[1]):
+            results[i] = results[parent[i]].dot(with_zeros(ch.hstack((
+                Rodrigues(pose[i, :]),      # rotation around bone endpoint
+                (self.J[i, :] - self.J[parent[i], :]).reshape((3, 1))     # bone
+            ))))
+
+        results = [results[i] for i in sorted(results.keys())]
+        results_global = results
+
+        # subtract rotated J position
+        results2 = [results[i] - (pack(
+            results[i].dot(ch.concatenate((self.J[i, :], [0]))))
+        ) for i in range(len(results))]
+        result = ch.dstack(results2)
+
+        return result, results_global
+
+    def compute_r(self):
+        return self.v.r
+
+    def compute_dr_wrt(self, wrt):
+        if wrt is not self.trans and wrt is not self.betas and wrt is not self.pose and wrt is not self.v_personal:
+            return None
+
+        return self.v.dr_wrt(wrt)
+
+
+def copy_smpl(smpl, model):
+    new = Smpl(model, betas=smpl.betas)
+    new.pose[:] = smpl.pose.r
+    new.trans[:] = smpl.trans.r
+
+    return new
+
+
+def joints_coco(smpl):
+    J = smpl.J_transformed
+    nose = smpl[VERT_NOSE]
+    ear_l = smpl[VERT_EAR_L]
+    ear_r = smpl[VERT_EAR_R]
+    eye_l = smpl[VERT_EYE_L]
+    eye_r = smpl[VERT_EYE_R]
+
+    shoulders_m = ch.sum(J[[14, 13]], axis=0) / 2.
+    neck = J[12] - 0.55 * (J[12] - shoulders_m)
+
+    return ch.vstack((
+        nose,
+        neck,
+        2.1 * (J[14] - shoulders_m) + neck,
+        J[[19, 21]],
+        2.1 * (J[13] - shoulders_m) + neck,
+        J[[18, 20]],
+        J[2] + 0.38 * (J[2] - J[1]),
+        J[[5, 8]],
+        J[1] + 0.38 * (J[1] - J[2]),
+        J[[4, 7]],
+        eye_r,
+        eye_l,
+        ear_r,
+        ear_l,
+    ))
+
+
+def model_params_in_camera_coords(trans, pose, J0, camera_t, camera_rt):
+    root = Rodrigues(np.matmul(Rodrigues(camera_rt).r, Rodrigues(pose[:3]).r)).r.reshape(-1)
+    pose[:3] = root
+
+    trans = (Rodrigues(camera_rt).dot(J0 + trans) - J0 + camera_t).r
+
+    return trans, pose
+
+
+if __name__ == '__main__':
+    smpl = Smpl(model='../vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl')
+    smpl.pose[:] = np.random.randn(72) * .2
+    smpl.pose[0] = np.pi
+    # smpl.v_personal[:] = np.random.randn(*smpl.shape) / 500.
+
+    # render test
+    from opendr.renderer import ColoredRenderer
+    from opendr.camera import ProjectPoints
+    from opendr.lighting import LambertianPointLight
+
+    rn = ColoredRenderer()
+
+    # Assign attributes to renderer
+    w, h = (640, 480)
+
+    rn.camera = ProjectPoints(v=smpl, rt=np.zeros(3), t=np.array([0, 0, 3.]), f=np.array([w, w]),
+                              c=np.array([w, h]) / 2., k=np.zeros(5))
+    rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
+    rn.set(v=smpl, f=smpl.f, bgcolor=np.zeros(3))
+
+    # Construct point light source
+    rn.vc = LambertianPointLight(
+        f=smpl.f,
+        v=rn.v,
+        num_verts=len(smpl),
+        light_pos=np.array([-1000, -1000, -2000]),
+        vc=np.ones_like(smpl) * .9,
+        light_color=np.array([1., 1., 1.]))
+
+    # Show it using OpenCV
+    import cv2
+
+    cv2.imshow('render_SMPL', rn.r)
+    print ('..Print any key while on the display window')
+    cv2.waitKey(0)
+    cv2.destroyAllWindows()

videoavatars/prepare_data/2djoints2hdf5.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import os
+import argparse
+import h5py
+import json
+import numpy as np
+
+from glob import glob
+from tqdm import tqdm
+import pdb
+"""
+
+This script stores OpenPose 2D keypoints from json files in the given directory in a compressed hdf5 file.
+
+Example:
+$ python 2djoints2hdf5.py dataset/subject/openpose_detections keypoints.hdf5
+
+"""
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('src_folder', type=str)
+parser.add_argument('target', type=str)
+
+args = parser.parse_args()
+
+out_file = args.target
+pose_dir = args.src_folder
+pose_files = sorted(glob(os.path.join(pose_dir, '*.json')))
+
+with h5py.File(out_file, 'w') as f:
+    poses_dset = f.create_dataset("keypoints", (len(pose_files), 54), 'f', chunks=True, compression="lzf") # 54
+    for i, pose_file in enumerate(tqdm(pose_files)):
+        with open(pose_file) as fp:
+            pose = np.array(json.load(fp)['people'][0]['pose_keypoints_2d'])
+            poses_dset[i] = pose

videoavatars/prepare_data/create_camera.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import argparse
+import cPickle as pkl
+
+"""
+
+This script creates a .pkl file using the given camera intrinsics.
+
+Example:
+$ python create_camera.py camera.pkl 1080 1080 -f 900.0 900.0
+
+"""
+
+parser = argparse.ArgumentParser()
+parser.add_argument('out', type=str, help="Output file (.pkl)")
+parser.add_argument('width', type=int, help="Frame width in px")
+parser.add_argument('height', type=int, help="Frame height in px")
+parser.add_argument('-f', type=float, nargs='*', help="Focal length in px (2,)")
+parser.add_argument('-c', type=float, nargs='*', help="Principal point in px (2,)")
+parser.add_argument('-k', type=float, nargs='*', help="Distortion coefficients (5,)")
+
+args = parser.parse_args()
+
+camera_data = {
+    'camera_t': np.zeros(3),
+    'camera_rt': np.zeros(3),
+    'camera_f': np.array([args.width, args.width]),
+    'camera_c': np.array([args.width, args.height]) / 2.,
+    'camera_k': np.zeros(5),
+    'width': args.width,
+    'height': args.height,
+}
+
+if args.f is not None:
+    if len(args.f) is not 2:
+        raise Exception('Focal length should be of shape (2,)')
+
+    camera_data['camera_f'] = np.array(args.f)
+
+if args.c is not None:
+    if len(args.c) is not 2:
+        raise Exception('Principal point should be of shape (2,)')
+
+    camera_data['camera_c'] = np.array(args.c)
+
+if args.k is not None:
+    if len(args.k) is not 5:
+        raise Exception('Distortion coefficients should be of shape (5,)')
+
+    camera_data['camera_k'] = np.array(args.k)
+
+with open(args.out, 'wb') as f:
+    pkl.dump(camera_data, f, protocol=2)

videoavatars/prepare_data/masks2hdf5.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import os
+import argparse
+import h5py
+import cv2
+import numpy as np
+
+from glob import glob
+from tqdm import tqdm
+
+"""
+
+This script stores image masks from a directory in a compressed hdf5 file.
+
+Example:
+$ python masks2hdf5.py dataset/subject/masks masks.hdf5
+
+"""
+
+parser = argparse.ArgumentParser()
+parser.add_argument('src', type=str)
+parser.add_argument('target', type=str)
+
+args = parser.parse_args()
+
+out_file = args.target
+mask_dir = args.src
+mask_files = sorted(glob(os.path.join(mask_dir, '*.png')) + glob(os.path.join(mask_dir, '*.jpg')))
+
+with h5py.File(out_file, 'w') as f:
+    dset = None
+
+    for i, silh_file in enumerate(tqdm(mask_files)):
+        silh = cv2.imread(silh_file, cv2.IMREAD_GRAYSCALE)
+
+        if dset is None:
+            dset = f.create_dataset("masks", (len(mask_files), silh.shape[0], silh.shape[1]), 'b', chunks=True, compression="lzf")
+
+        _, silh = cv2.threshold(silh, 100, 255, cv2.THRESH_BINARY)
+        dset[i] = silh.astype(np.bool)

videoavatars/render/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+

videoavatars/render/camera.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import cv2
+import numpy as np
+import chumpy as ch
+from chumpy import Ch, depends_on
+from chumpy.utils import col
+from opendr.geometry import Rodrigues
+
+
+class OrthoProjectPoints(Ch):
+    terms = 'near', 'far', 'width', 'height'
+    dterms = 'v', 'rt', 't', 'left', 'right', 'bottom', 'top'
+
+    def compute_r(self):
+        return self.r_and_derivatives.r
+
+    def compute_dr_wrt(self, wrt):
+        if wrt not in [self.v, self.rt, self.t, self.left, self.right, self.bottom, self.top]:
+            return None
+
+        return self.r_and_derivatives.dr_wrt(wrt)
+
+    @depends_on('t', 'rt')
+    def view_mtx(self):
+        R = cv2.Rodrigues(self.rt.r)[0]
+        return np.hstack((R, col(self.t.r)))
+
+    @property
+    def r_and_derivatives(self):
+        tmp = self.v.dot(Rodrigues(self.rt)) + self.t
+
+        return ch.hstack((
+            col(2. / (self.right - self.left) * tmp[:, 0] - (self.right + self.left) / (self.right - self.left) + 1.) * self.width / 2.,
+            col(2. / (self.bottom - self.top) * tmp[:, 1] - (self.bottom + self.top) / (self.bottom - self.top) + 1.) * self.height / 2.,
+        ))
+

videoavatars/render/renderer.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import platform
+import numpy as np
+
+from chumpy import Ch, depends_on
+from opendr.renderer import BaseRenderer, ColoredRenderer, TexturedRenderer
+from opendr.renderer import draw_edge_visibility, draw_boundary_images, draw_boundaryid_image
+
+if platform.system() == 'Darwin':
+    from opendr.contexts.ctx_mac import OsContext
+else:
+    from opendr.contexts.ctx_mesa import OsContext
+from opendr.contexts._constants import *
+
+
+class OrthoBaseRenderer(BaseRenderer):
+    terms = ['f', 'overdraw']
+    dterms = ['ortho', 'v']
+
+    @property
+    def v(self):
+        return self.ortho.v
+
+    @v.setter
+    def v(self, newval):
+        self.ortho.v = newval
+
+    @depends_on('f', 'ortho', 'overdraw')
+    def barycentric_image(self):
+        return super(OrthoBaseRenderer, self).barycentric_image
+
+    @depends_on(terms+dterms)
+    def boundaryid_image(self):
+        self._call_on_changed()
+        return draw_boundaryid_image(self.glb, self.v.r, self.f, self.vpe, self.fpe, self.ortho)
+
+    @depends_on('f', 'ortho', 'overdraw')
+    def visibility_image(self):
+        return super(OrthoBaseRenderer, self).visibility_image
+
+    @depends_on('f', 'ortho')
+    def edge_visibility_image(self):
+        self._call_on_changed()
+        return draw_edge_visibility(self.glb, self.v.r, self.vpe, self.f)
+
+
+class OrthoColoredRenderer(OrthoBaseRenderer, ColoredRenderer):
+    terms = 'f', 'background_image', 'overdraw', 'num_channels'
+    dterms = 'vc', 'ortho', 'bgcolor'
+
+    def compute_r(self):
+        return self.color_image
+
+    def compute_dr_wrt(self, wrt):
+        raise NotImplementedError
+
+    def on_changed(self, which):
+        if 'ortho' in which:
+            w = self.ortho.width
+            h = self.ortho.height
+            self.glf = OsContext(np.int(w), np.int(h), typ=GL_FLOAT)
+            _setup_ortho(self.glf, self.ortho.left.r, self.ortho.right.r, self.ortho.bottom.r, self.ortho.top.r,
+                         self.ortho.near, self.ortho.far, self.ortho.view_mtx)
+            self.glf.Viewport(0, 0, w, h)
+            self.glb = OsContext(np.int(w), np.int(h), typ=GL_UNSIGNED_BYTE)
+            self.glb.Viewport(0, 0, w, h)
+            _setup_ortho(self.glb, self.ortho.left.r, self.ortho.right.r, self.ortho.bottom.r, self.ortho.top.r,
+                         self.ortho.near, self.ortho.far, self.ortho.view_mtx)
+
+        if not hasattr(self, 'num_channels'):
+            self.num_channels = 3
+
+        if not hasattr(self, 'bgcolor'):
+            self.bgcolor = Ch(np.array([.5] * self.num_channels))
+            which.add('bgcolor')
+
+        if not hasattr(self, 'overdraw'):
+            self.overdraw = True
+
+        if 'bgcolor' in which:
+            self.glf.ClearColor(self.bgcolor.r[0], self.bgcolor.r[1 % self.num_channels],
+                                self.bgcolor.r[2 % self.num_channels], 1.)
+
+    @depends_on('f', 'ortho', 'vc')
+    def boundarycolor_image(self):
+        return self.draw_boundarycolor_image(with_vertex_colors=True)
+
+    @depends_on('f', 'ortho')
+    def boundary_images(self):
+        self._call_on_changed()
+        return draw_boundary_images(self.glb, self.v.r, self.f, self.vpe, self.fpe, self.ortho)
+
+    @depends_on(terms+dterms)
+    def color_image(self):
+        return super(OrthoColoredRenderer, self).color_image
+
+    @property
+    def shape(self):
+        return (self.ortho.height, self.ortho.width, 3)
+
+
+class OrthoTexturedRenderer(OrthoColoredRenderer, TexturedRenderer):
+    terms = 'f', 'ft', 'background_image', 'overdraw', 'tex_filter_mag', 'tex_filter_min'
+    dterms = 'vc', 'ortho', 'bgcolor', 'texture_image', 'vt'
+
+    def compute_dr_wrt(self, wrt):
+        raise NotImplementedError
+
+    def on_changed(self, which):
+        OrthoColoredRenderer.on_changed(self, which)
+
+        # have to redo if ortho changes, b/c ortho triggers new context
+        if 'texture_image' in which or 'ortho' in which:
+            gl = self.glf
+            texture_data = np.array(self.texture_image * 255., dtype='uint8', order='C')
+            tmp = np.zeros(1, dtype=np.uint32)
+
+            self.release_textures()
+            gl.GenTextures(1, tmp)
+
+            self.textureID = tmp[0]
+            gl.BindTexture(GL_TEXTURE_2D, self.textureID)
+
+            gl.TexImage2Dub(GL_TEXTURE_2D, 0, GL_RGB, texture_data.shape[1], texture_data.shape[0], 0, GL_RGB,
+                            texture_data.ravel())
+            gl.GenerateMipmap(GL_TEXTURE_2D)
+
+    def release_textures(self):
+        if hasattr(self, 'textureID'):
+            arr = np.asarray(np.array([self.textureID]), np.uint32)
+            self.glf.DeleteTextures(arr)
+
+    def texture_mapping_on(self, gl, with_vertex_colors):
+        gl.Enable(GL_TEXTURE_2D)
+        gl.BindTexture(GL_TEXTURE_2D, self.textureID)
+        gl.TexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
+        gl.TexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR)
+        gl.TexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE if with_vertex_colors else GL_REPLACE)
+        gl.EnableClientState(GL_TEXTURE_COORD_ARRAY)
+
+    @depends_on(dterms+terms)
+    def boundaryid_image(self):
+        return super(OrthoTexturedRenderer, self).boundaryid_image
+
+    @depends_on(terms+dterms)
+    def color_image(self):
+        self.glf.BindTexture(GL_TEXTURE_2D, self.textureID)
+        return super(OrthoTexturedRenderer, self).color_image
+
+    @depends_on(terms+dterms)
+    def boundarycolor_image(self):
+        self.glf.BindTexture(GL_TEXTURE_2D, self.textureID)
+        return super(OrthoTexturedRenderer, self).boundarycolor_image
+
+    @property
+    def shape(self):
+        return (self.ortho.height, self.ortho.width, 3)
+
+    @depends_on('vt', 'ft')
+    def mesh_tex_coords(self):
+        ftidxs = self.ft.ravel()
+        data = np.asarray(self.vt.r[ftidxs].astype(np.float32)[:, 0:2], np.float32, order='C')
+        data[:, 1] = 1.0 - 1.0 * data[:, 1]
+        return data
+
+
+def _setup_ortho(gl, l, r, b, t, near, far, view_matrix):
+    gl.MatrixMode(GL_PROJECTION)
+    gl.LoadIdentity()
+    gl.Ortho(l, r, t, b, near, far)  # top and bottom switched for opencv coordinate system
+
+    gl.MatrixMode(GL_MODELVIEW)
+    gl.LoadIdentity()
+    gl.Rotatef(180, 1, 0, 0)
+
+    view_mtx = np.asarray(np.vstack((view_matrix, np.array([0, 0, 0, 1]))), np.float32, order='F')
+    gl.MultMatrixf(view_mtx)
+
+    gl.Enable(GL_DEPTH_TEST)
+    gl.PolygonMode(GL_BACK, GL_FILL)
+    gl.Disable(GL_LIGHTING)
+    gl.Disable(GL_CULL_FACE)
+    gl.PixelStorei(GL_PACK_ALIGNMENT, 1)
+    gl.PixelStorei(GL_UNPACK_ALIGNMENT, 1)
+
+    gl.UseProgram(0)

videoavatars/requirements.txt

@@ -0,0 +1,8 @@
+numpy>=1.11.0
+opendr>=0.76
+
+scipy>=0.19.0
+matplotlib==2.*
+h5py==2.7.*
+tqdm==4.*
+chumpy>=0.67

videoavatars/run_step1.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#ca videoavatar
+
+if [ "$#" -le 1 ]; then
+    echo "usage: run_step1.sh <path_to_subject_directory> <output_directory> [options]" >&2
+    exit 1
+fi
+
+SUBJ="$1"
+
+OUT="$2"
+gender="$3"
+
+if [[ $gender = "female" ]]; then
+    MODEL='--model vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl'
+    echo "use female smpl model"
+elif [[ $gender = "male" ]]; then
+    MODEL='--model vendor/smpl/models/basicmodel_m_lbs_10_207_0_v1.0.0.pkl'
+    echo "use male smpl model"
+else
+    echo "Invalid gender: $gender"
+    exit 1
+fi
+
+echo $MODEL
+python step1_pose.py \
+    $SUBJ/keypoints.hdf5 $SUBJ/masks.hdf5 $SUBJ/camera.pkl $OUT/reconstructed_poses.hdf5  $MODEL #${@:3}

videoavatars/run_step2.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ "$#" -le 1 ]; then
+    echo "usage: run_step2.sh <path_to_subject_directory> <output_directory> [options]" >&2
+    exit 1
+fi
+
+SUBJ="$1"
+OUT="$2"
+
+#if [[ $SUBJ = *"female"* ]]; then
+  MODEL='--model vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl'
+#fi
+
+python step2_consensus.py $SUBJ/reconstructed_poses.hdf5 $SUBJ/masks.hdf5 $SUBJ/camera.pkl $OUT/consensus.pkl --obj_out $OUT/consensus.obj $MODEL ${@:3}

videoavatars/run_step3.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ "$#" -le 1 ]; then
+    echo "usage: run_step3.sh <path_to_subject_directory> <output_directory> [options]" >&2
+    exit 1
+fi
+
+SUBJ="$1"
+OUT="$2"
+
+if [[ $SUBJ = *"female"* ]]; then
+  MODEL='--model vendor/smpl/models/basicModel_f_lbs_10_207_0_v1.0.0.pkl'
+fi
+
+python step3_texture.py $SUBJ/consensus.pkl  $SUBJ/camera.pkl $SUBJ/$(basename $SUBJ).mp4 $SUBJ/reconstructed_poses.hdf5 $SUBJ/masks.hdf5 $OUT/tex-$(basename $SUBJ).jpg $MODEL ${@:3}

videoavatars/step1_pose.py

@@ -0,0 +1,377 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import cv2, pdb
+import h5py
+import argparse
+import numpy as np
+import chumpy as ch
+import cPickle as pkl
+
+from opendr.camera import ProjectPoints
+from opendr.lighting import LambertianPointLight
+from opendr.renderer import ColoredRenderer
+from opendr.filters import gaussian_pyramid
+
+from util import im
+from util.logger import log
+from lib.frame import FrameData
+from models.smpl import Smpl, copy_smpl, joints_coco
+from models.bodyparts import faces_no_hands
+
+from vendor.smplify.sphere_collisions import SphereCollisions
+from vendor.smplify.robustifiers import GMOf
+
+
+def get_cb(viz_rn, f):
+    if viz_rn is not None:
+        viz_rn.set(v=f.smpl, background_image=np.dstack((f.mask, f.mask, f.mask)))
+        viz_rn.vc.set(v=f.smpl)
+
+        def cb(_):
+            debug = np.array(viz_rn.r)
+
+            for j in f.J_proj.r:
+                cv2.circle(debug, tuple(j.astype(np.int)), 3, (0, 0, 0.8), -1)
+            for j in f.keypoints[:, :2]:
+                cv2.circle(debug, tuple(j.astype(np.int)), 3, (0, 0.8, 0), -1)
+
+            im.show(debug, id='pose', waittime=1)
+    else:
+        cb = None
+
+    return cb
+
+
+def collision_obj(smpl, regs):
+    sp = SphereCollisions(pose=smpl.pose, betas=smpl.betas, model=smpl, regs=regs)
+    sp.no_hands = True
+
+    return sp
+
+
+def pose_prior_obj(smpl, prior_data):
+    return (smpl.pose[3:] - prior_data['mean']).reshape(1, -1).dot(prior_data['prec'])
+
+
+def height_predictor(b2m, betas):
+    return ch.hstack((betas.reshape(1, -1), [[1]])).dot(b2m)
+
+
+def init(frames, body_height, b2m, viz_rn):
+    betas = frames[0].smpl.betas
+
+    E_height = None
+    if body_height is not None:
+         E_height = height_predictor(b2m, betas) - body_height * 1000.
+
+    # first get a rough pose for all frames individually
+    for i, f in enumerate(frames):
+        if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.:
+            if f.keypoints[2, 0] > f.keypoints[5, 0]:
+                f.smpl.pose[0] = 0
+                f.smpl.pose[2] = np.pi
+            # pdb.set_trace()
+            E_init = {
+                'init_pose_{}'.format(i): f.pose_obj[[0, 2, 5, 8, 11]]
+            }
+
+            x0 = [f.smpl.trans, f.smpl.pose[:3]]
+
+            if E_height is not None and i == 0:
+                E_init['height'] = E_height
+                E_init['betas'] = betas
+                x0.append(betas)
+
+            ch.minimize(
+                E_init,
+                x0,
+                method='dogleg',
+                options={
+                    'e_3': .01,
+                },
+                callback=get_cb(viz_rn, f)
+            )
+
+    weights = zip(
+        [5., 4.5, 4.],
+        [5., 4., 3.]
+    )
+
+    E_betas = betas - betas.r
+
+    for w_prior, w_betas in weights:
+        x0 = [betas]
+
+        E = {
+            'betas': E_betas * w_betas,
+        }
+
+        if E_height is not None:
+            E['height'] = E_height
+
+        for i, f in enumerate(frames):
+            if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.:
+                x0.extend([f.smpl.pose[range(21) + range(27, 30) + range(36, 60)], f.smpl.trans])
+                E['pose_{}'.format(i)] = f.pose_obj
+                E['prior_{}'.format(i)] = f.pose_prior_obj * w_prior
+
+        ch.minimize(
+            E,
+            x0,
+            method='dogleg',
+            options={
+                'e_3': .01,
+            },
+            callback=get_cb(viz_rn, frames[0])
+        )
+
+
+def reinit_frame(frame, null_pose, nohands, viz_rn):
+
+    if (np.sum(frame.pose_obj.r ** 2) > 625 or np.sum(frame.pose_prior_obj.r ** 2) > 75)\
+            and np.sum(frame.keypoints[[0, 2, 5, 8, 11], 2]) > 3.:
+
+        log.info('Tracking error too large. Re-init frame...')
+
+        x0 = [frame.smpl.pose[:3], frame.smpl.trans]
+
+        frame.smpl.pose[3:] = null_pose
+        if frame.keypoints[2, 0] > frame.keypoints[5, 0]:
+            frame.smpl.pose[0] = 0
+            frame.smpl.pose[2] = np.pi
+
+        E = {
+            'init_pose': frame.pose_obj[[0, 2, 5, 8, 11]],
+        }
+
+        ch.minimize(
+            E,
+            x0,
+            method='dogleg',
+            options={
+                'e_3': .1,
+            },
+            callback=get_cb(viz_rn, frame)
+        )
+
+        E = {
+            'pose': GMOf(frame.pose_obj, 100),
+            'prior': frame.pose_prior_obj * 8.,
+        }
+
+        x0 = [frame.smpl.trans]
+
+        if nohands:
+            x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)])
+        else:
+            x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)])
+
+        ch.minimize(
+            E,
+            x0,
+            method='dogleg',
+            options={
+                'e_3': .01,
+            },
+            callback=get_cb(viz_rn, frame)
+        )
+
+
+def fit_pose(frame, last_smpl, frustum, nohands, viz_rn):
+
+    if nohands:
+        faces = faces_no_hands(frame.smpl.f)
+    else:
+        faces = frame.smpl.f
+
+    dst_type = cv2.cv.CV_DIST_L2 if cv2.__version__[0] == '2' else cv2.DIST_L2
+
+    dist_i = cv2.distanceTransform(np.uint8(frame.mask * 255), dst_type, 5) - 1
+    dist_i[dist_i < 0] = 0
+    dist_i[dist_i > 50] = 50
+    dist_o = cv2.distanceTransform(255 - np.uint8(frame.mask * 255), dst_type, 5)
+    dist_o[dist_o > 50] = 50
+
+    rn_m = ColoredRenderer(camera=frame.camera, v=frame.smpl, f=faces, vc=np.ones_like(frame.smpl), frustum=frustum,
+                           bgcolor=0, num_channels=1)
+
+    E = {
+        'mask': gaussian_pyramid(rn_m * dist_o * 100. + (1 - rn_m) * dist_i, n_levels=4, normalization='size') * 80.,
+        '2dpose': GMOf(frame.pose_obj, 100),
+        'prior': frame.pose_prior_obj * 4.,
+        'sp': frame.collision_obj * 1e3,
+    }
+
+    if last_smpl is not None:
+        E['last_pose'] = GMOf(frame.smpl.pose - last_smpl.pose, 0.05) * 50.
+        E['last_trans'] = GMOf(frame.smpl.trans - last_smpl.trans, 0.05) * 50.
+
+    if nohands:
+        x0 = [frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)], frame.smpl.trans]
+    else:
+        x0 = [frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)], frame.smpl.trans]
+
+    ch.minimize(
+        E,
+        x0,
+        method='dogleg',
+        options={
+            'e_3': .01,
+        },
+        callback=get_cb(viz_rn, frame)
+    )
+
+
+def main(keypoint_file, masks_file, camera_file, out, model_file, prior_file, resize,
+         body_height, nohands, display):
+
+    # load data
+    with open(model_file, 'rb') as fp:
+        model_data = pkl.load(fp)
+
+    with open(camera_file, 'rb') as fp:
+        camera_data = pkl.load(fp)
+
+    with open(prior_file, 'rb') as fp:
+        prior_data = pkl.load(fp)
+
+    if 'basicModel_f' in model_file:
+        regs = np.load('vendor/smplify/models/regressors_locked_normalized_female.npz')
+        b2m = np.load('assets/b2m_f.npy')
+    else:
+        regs = np.load('vendor/smplify/models/regressors_locked_normalized_male.npz')
+        b2m = np.load('assets/b2m_m.npy')
+
+    keypoints = h5py.File(keypoint_file, 'r')['keypoints']
+    masks = h5py.File(masks_file, 'r')['masks']
+    num_frames = masks.shape[0]
+
+    # init
+    base_smpl = Smpl(model_data)
+    base_smpl.trans[:] = np.array([0, 0, 3])
+    base_smpl.pose[0] = np.pi
+    base_smpl.pose[3:] = prior_data['mean']
+
+    camera = ProjectPoints(t=np.zeros(3), rt=np.zeros(3), c=camera_data['camera_c'] * resize,
+                           f=camera_data['camera_f'] * resize, k=camera_data['camera_k'], v=base_smpl)
+    frustum = {'near': 0.1, 'far': 1000.,
+               'width': int(camera_data['width'] * resize), 'height': int(camera_data['height'] * resize)}
+
+    if display:
+        debug_cam = ProjectPoints(v=base_smpl, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k)
+        debug_light = LambertianPointLight(f=base_smpl.f, v=base_smpl, num_verts=len(base_smpl), light_pos=np.zeros(3),
+                                           vc=np.ones(3), light_color=np.ones(3))
+        debug_rn = ColoredRenderer(camera=debug_cam, v=base_smpl, f=base_smpl.f, vc=debug_light, frustum=frustum)
+    else:
+        debug_rn = None
+
+    # generic frame loading function
+    def create_frame(i, smpl, copy=True):
+        f = FrameData()
+
+        f.smpl = copy_smpl(smpl, model_data) if copy else smpl
+        f.camera = ProjectPoints(v=f.smpl, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k)
+
+        f.keypoints = np.array(keypoints[i]).reshape(-1, 3) * np.array([resize, resize, 1])
+        f.J = joints_coco(f.smpl)
+        f.J_proj = ProjectPoints(v=f.J, t=camera.t, rt=camera.rt, c=camera.c, f=camera.f, k=camera.k)
+        f.mask = cv2.resize(np.array(masks[i], dtype=np.float32), (0, 0),
+                            fx=resize, fy=resize, interpolation=cv2.INTER_NEAREST)
+
+        f.collision_obj = collision_obj(f.smpl, regs)
+        f.pose_prior_obj = pose_prior_obj(f.smpl, prior_data)
+        f.pose_obj = (f.J_proj - f.keypoints[:, :2]) * f.keypoints[:, 2].reshape(-1, 1)
+
+        return f
+
+    base_frame = create_frame(0, base_smpl, copy=False)
+
+    # get betas from 5 frames
+    log.info('Initial fit')
+
+    num_init = 5
+    indices_init = np.ceil(np.arange(num_init) * num_frames * 1. / num_init).astype(np.int)
+
+    init_frames = [base_frame]
+    for i in indices_init[1:]:
+        init_frames.append(create_frame(i, base_smpl))
+
+    init(init_frames, body_height, b2m, debug_rn)
+
+    # get pose frame by frame
+    with h5py.File(out, 'w') as fp:
+        last_smpl = None
+        poses_dset = fp.create_dataset("pose", (num_frames, 72), 'f', chunks=True, compression="lzf")
+        trans_dset = fp.create_dataset("trans", (num_frames, 3), 'f', chunks=True, compression="lzf")
+        betas_dset = fp.create_dataset("betas", (10,), 'f', chunks=True, compression="lzf")
+
+        for i in xrange(num_frames):
+            if i == 0:
+                current_frame = base_frame
+            else:
+                current_frame = create_frame(i, last_smpl)
+
+            log.info('Fit frame {}'.format(i))
+            # re-init if necessary
+            reinit_frame(current_frame, prior_data['mean'], nohands, debug_rn)
+            # final fit
+            fit_pose(current_frame, last_smpl, frustum, nohands, debug_rn)
+
+            poses_dset[i] = current_frame.smpl.pose.r
+            trans_dset[i] = current_frame.smpl.trans.r
+
+            if i == 0:
+                betas_dset[:] = current_frame.smpl.betas.r
+
+            last_smpl = current_frame.smpl
+
+    log.info('Done.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        'keypoint_file',
+        type=str,
+        help="File that contains 2D keypoint detections")
+    parser.add_argument(
+        'masks_file',
+        type=str,
+        help="File that contains segmentations")
+    parser.add_argument(
+        'camera',
+        type=str,
+        help="pkl file that contains camera settings")
+    parser.add_argument(
+        'out',
+        type=str,
+        help="Out file path")
+    parser.add_argument(
+        '--model', '-m',
+        default='vendor/smpl/models/basicmodel_m_lbs_10_207_0_v1.1.0.pkl',
+        help='Path to SMPL model')
+    parser.add_argument(
+        '--prior', '-p',
+        default='assets/prior_a_pose.pkl',
+        help='Path to pose prior')
+    parser.add_argument(
+        '--resize', '-r', default=0.5, type=float,
+        help="Resize factor")
+    parser.add_argument(
+        '--body_height', '-bh', default=None, type=float,
+        help="Height of the subject in meters (optional)")
+    parser.add_argument(
+        '--nohands', '-nh',
+        action='store_true',
+        help="Exclude hands from optimization")
+    parser.add_argument(
+        '--display', '-d',
+        action='store_true',
+        help="Enable visualization")
+
+    args = parser.parse_args()
+
+    main(args.keypoint_file, args.masks_file, args.camera, args.out, args.model, args.prior, args.resize,
+         args.body_height, args.nohands, args.display)

videoavatars/step2_consensus.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import h5py
+import argparse
+import numpy as np
+import chumpy as ch
+import cPickle as pkl
+
+from opendr.camera import ProjectPoints
+from opendr.renderer import BoundaryRenderer, ColoredRenderer
+from tqdm import tqdm
+
+from util import im, mesh
+from util.logger import log
+from lib.frame import setup_frame_rays
+from lib.rays import ray_objective
+from lib.geometry import laplacian
+from lib.ch import sp_dot
+from models.smpl import Smpl
+from models.bodyparts import faces_no_hands, regularize_laplace, regularize_model, regularize_symmetry
+
+
+def get_cb(frame, base_smpl, camera, frustum):
+    viz_mask = frame.mask / 255.
+    base_smpl.pose[:] = frame.pose
+    camera.t[:] = frame.trans
+    camera.rt[:] = 0
+
+    rn = ColoredRenderer(camera=camera, v=base_smpl, f=base_smpl.f, vc=np.ones_like(base_smpl),
+                         frustum=frustum, bgcolor=0, num_channels=1)
+
+    def cb(_):
+        silh_diff = (rn.r - viz_mask + 1) / 2.
+        im.show(silh_diff, waittime=1)
+
+    return cb
+
+
+def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display):
+    if nohands:
+        faces = faces_no_hands(base_smpl.f)
+    else:
+        faces = base_smpl.f
+
+    vis_rn_b = BoundaryRenderer(camera=camera, frustum=frustum, f=faces, num_channels=1)
+    vis_rn_m = ColoredRenderer(camera=camera, frustum=frustum, f=faces, vc=np.zeros_like(base_smpl), bgcolor=1,
+                               num_channels=1)
+
+    model_template = Smpl(model_data)
+    model_template.betas[:] = base_smpl.betas.r
+
+    g_laplace = regularize_laplace()
+    g_model = regularize_model()
+    g_symmetry = regularize_symmetry()
+
+    for step, (w_laplace, w_model, w_symmetry, sigma) in enumerate(zip(
+            np.linspace(6.5, 4.0, icp_count) if naked else np.linspace(4.0, 2.0, icp_count),
+            np.linspace(0.9, 0.6, icp_count) if naked else np.linspace(0.6, 0.3, icp_count),
+            np.linspace(3.6, 1.8, icp_count),
+            np.linspace(0.06, 0.003, icp_count),
+    )):
+        log.info('# Step {}'.format(step))
+
+        L = laplacian(model_template.r, base_smpl.f)
+        delta = L.dot(model_template.r)
+
+        w_laplace *= g_laplace.reshape(-1, 1)
+        w_model *= g_model.reshape(-1, 1)
+        w_symmetry *= g_symmetry.reshape(-1, 1)
+
+        E = {
+            'laplace': (sp_dot(L, base_smpl.v_shaped_personal) - delta) * w_laplace,
+            'model': (base_smpl.v_shaped_personal - model_template) * w_model,
+            'symmetry': (base_smpl.v_personal + np.array([1, -1, -1])
+                         * base_smpl.v_personal[model_data['vert_sym_idxs']]) * w_symmetry,
+        }
+
+        log.info('## Matching rays with contours')
+        for current, f in enumerate(tqdm(frames)):
+            E['silh_{}'.format(current)] = ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m)
+
+        log.info('## Run optimization')
+        ch.minimize(
+            E,
+            [base_smpl.v_personal, model_template.betas],
+            method='dogleg',
+            options={'maxiter': 15, 'e_3': 0.001},
+            callback=get_cb(frames[0], base_smpl, camera, frustum) if display else None
+        )
+
+
+def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model_file, first_frame, last_frame,
+         nohands, naked, display):
+
+    # load data
+    with open(model_file, 'rb') as fp:
+        model_data = pkl.load(fp)
+
+    with open(camera_file, 'rb') as fp:
+        camera_data = pkl.load(fp)
+
+    pose_data = h5py.File(pose_file, 'r')
+    poses = pose_data['pose'][first_frame:last_frame]
+    trans = pose_data['trans'][first_frame:last_frame]
+    masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame]
+    num_frames = masks.shape[0]
+
+    indices_consensus = np.ceil(np.arange(num) * num_frames * 1. / num).astype(np.int)
+
+    # init
+    base_smpl = Smpl(model_data)
+    base_smpl.betas[:] = np.array(pose_data['betas'], dtype=np.float32)
+
+    camera = ProjectPoints(t=np.zeros(3), rt=np.zeros(3), c=camera_data['camera_c'],
+                           f=camera_data['camera_f'], k=camera_data['camera_k'], v=base_smpl)
+    camera_t = camera_data['camera_t']
+    camera_rt = camera_data['camera_rt']
+    frustum = {'near': 0.1, 'far': 1000., 'width': int(camera_data['width']), 'height': int(camera_data['height'])}
+    frames = []
+
+    for i in indices_consensus:
+        log.info('Set up frame {}...'.format(i))
+
+        mask = np.array(masks[i] * 255, dtype=np.uint8)
+        pose_i = np.array(poses[i], dtype=np.float32)
+        trans_i = np.array(trans[i], dtype=np.float32)
+
+        frames.append(setup_frame_rays(base_smpl, camera, camera_t, camera_rt, pose_i, trans_i, mask))
+
+    log.info('Set up complete.')
+    log.info('Begin consensus fit...')
+    fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display)
+
+    with open(out, 'wb') as fp:
+        pkl.dump({
+            'v_personal': base_smpl.v_personal.r,
+            'betas': base_smpl.betas.r,
+        }, fp, protocol=2)
+
+    if obj_out is not None:
+        base_smpl.pose[:] = 0
+        vt = np.load('assets/basicModel_vt.npy')
+        ft = np.load('assets/basicModel_ft.npy')
+        mesh.write(obj_out, base_smpl.r, base_smpl.f, vt=vt, ft=ft)
+
+    log.info('Done.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'pose_file',
+        type=str,
+        help="File that contains poses")
+    parser.add_argument(
+        'masks_file',
+        type=str,
+        help="File that contains segmentations")
+    parser.add_argument(
+        'camera',
+        type=str,
+        help="pkl file that contains camera settings")
+    parser.add_argument(
+        'out',
+        type=str,
+        help="Out file path")
+    parser.add_argument(
+        '--obj_out', '-oo',
+        default=None,
+        help='obj out file name (optional)')
+    parser.add_argument(
+        '--num', '-n', default=120, type=int,
+        help="Number of used frames")
+    parser.add_argument(
+        '--icp', '-i', default=3, type=int,
+        help="ICP Iterations")
+    parser.add_argument(
+        '--model', '-m',
+        default='vendor/smpl/models/basicmodel_m_lbs_10_207_0_v1.0.0.pkl',
+        help='Path to SMPL model')
+    parser.add_argument(
+        '--first_frame', '-f', default=0, type=int,
+        help="First frame to use")
+    parser.add_argument(
+        '--last_frame', '-l', default=2000, type=int,
+        help="Last frame to use")
+    parser.add_argument(
+        '--nohands', '-nh',
+        action='store_true',
+        help="Exclude hands from optimization")
+    parser.add_argument(
+        '--naked', '-nk',
+        action='store_true',
+        help="Person wears (almost) no clothing")
+    parser.add_argument(
+        '--display', '-d',
+        action='store_true',
+        help="Enable visualization")
+
+    args = parser.parse_args()
+
+    main(args.pose_file, args.masks_file, args.camera, args.out, args.obj_out, args.num, args.icp, args.model,
+         args.first_frame, args.last_frame, args.nohands, args.naked, args.display)

videoavatars/step3_texture.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import cv2
+import h5py
+import argparse
+import numpy as np
+import cPickle as pkl
+
+from opendr.renderer import ColoredRenderer
+from opendr.camera import ProjectPoints
+from opendr.geometry import VertNormals
+from tex.iso import Isomapper, IsoColoredRenderer
+
+from util import im
+from util.logger import log
+from models.smpl import Smpl
+
+
+def main(consensus_file, camera_file, video_file, pose_file, masks_file, out, model_file, resolution, num,
+         first_frame, last_frame, display):
+    # load data
+    with open(model_file, 'rb') as fp:
+        model_data = pkl.load(fp)
+
+    with open(camera_file, 'rb') as fp:
+        camera_data = pkl.load(fp)
+
+    with open(consensus_file, 'rb') as fp:
+        consensus_data = pkl.load(fp)
+
+    pose_data = h5py.File(pose_file, 'r')
+    poses = pose_data['pose'][first_frame:last_frame]
+    trans = pose_data['trans'][first_frame:last_frame]
+    masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame]
+    num_frames = masks.shape[0]
+    indices_texture = np.ceil(np.arange(num) * num_frames * 1. / num).astype(np.int)
+
+    vt = np.load('assets/basicModel_vt.npy')
+    ft = np.load('assets/basicModel_ft.npy')
+
+    # init
+    base_smpl = Smpl(model_data)
+    base_smpl.betas[:] = consensus_data['betas']
+    base_smpl.v_personal[:] = consensus_data['v_personal']
+
+    bgcolor = np.array([1., 0.2, 1.])
+    iso = Isomapper(vt, ft, base_smpl.f, resolution, bgcolor=bgcolor)
+    iso_vis = IsoColoredRenderer(vt, ft, base_smpl.f, resolution)
+    camera = ProjectPoints(t=camera_data['camera_t'], rt=camera_data['camera_rt'], c=camera_data['camera_c'],
+                           f=camera_data['camera_f'], k=camera_data['camera_k'], v=base_smpl)
+    frustum = {'near': 0.1, 'far': 1000., 'width': int(camera_data['width']), 'height': int(camera_data['height'])}
+    rn_vis = ColoredRenderer(f=base_smpl.f, frustum=frustum, camera=camera, num_channels=1)
+
+    cap = cv2.VideoCapture(video_file)
+    for _ in range(first_frame):
+        cap.grab()
+
+    # get part-textures
+    i = first_frame
+
+    tex_agg = np.zeros((resolution, resolution, 25, 3))
+    tex_agg[:] = np.nan
+    normal_agg = np.ones((resolution, resolution, 25)) * 0.2
+
+    vn = VertNormals(f=base_smpl.f, v=base_smpl)
+    static_indices = np.indices((resolution, resolution))
+
+    while cap.isOpened() and i < indices_texture[-1]:
+        if i in indices_texture:
+            log.info('Getting part texture from frame {}...'.format(i))
+            _, frame = cap.read()
+
+            mask = np.array(masks[i], dtype=np.uint8)
+            pose_i = np.array(poses[i], dtype=np.float32)
+            trans_i = np.array(trans[i], dtype=np.float32)
+
+            base_smpl.pose[:] = pose_i
+            base_smpl.trans[:] = trans_i
+
+            # which faces have been seen and are projected into the silhouette?
+            visibility = rn_vis.visibility_image.ravel()
+            visible = np.nonzero(visibility != 4294967295)[0]
+
+            proj = camera.r
+            in_viewport = np.logical_and(
+                np.logical_and(np.round(camera.r[:, 0]) >= 0, np.round(camera.r[:, 0]) < frustum['width']),
+                np.logical_and(np.round(camera.r[:, 1]) >= 0, np.round(camera.r[:, 1]) < frustum['height']),
+            )
+            in_mask = np.zeros(camera.shape[0], dtype=np.bool)
+            idx = np.round(proj[in_viewport][:, [1, 0]].T).astype(np.int).tolist()
+            in_mask[in_viewport] = mask[idx]
+
+            faces_in_mask = np.where(np.min(in_mask[base_smpl.f], axis=1))[0]
+            visible_faces = np.intersect1d(faces_in_mask, visibility[visible])
+
+            # get the current unwrap
+            part_tex = iso.render(frame / 255., camera, visible_faces)
+
+            # angle under which the texels have been seen
+            points = np.hstack((proj, np.ones((proj.shape[0], 1))))
+            points3d = camera.unproject_points(points)
+            points3d /= np.linalg.norm(points3d, axis=1).reshape(-1, 1)
+            alpha = np.sum(points3d * -vn.r, axis=1).reshape(-1, 1)
+            alpha[alpha < 0] = 0
+            iso_normals = iso_vis.render(alpha)[:, :, 0]
+            iso_normals[np.all(part_tex == bgcolor, axis=2)] = 0
+
+            # texels to consider
+            part_mask = np.zeros((resolution, resolution))
+            min_normal = np.min(normal_agg, axis=2)
+            part_mask[iso_normals > min_normal] = 1.
+
+            # update best seen texels
+            where = np.argmax(np.atleast_3d(iso_normals) - normal_agg, axis=2)
+
+            idx = np.dstack((static_indices[0], static_indices[1], where))[part_mask == 1]
+            tex_agg[list(idx[:, 0]), list(idx[:, 1]), list(idx[:, 2])] = part_tex[part_mask == 1]
+            normal_agg[list(idx[:, 0]), list(idx[:, 1]), list(idx[:, 2])] = iso_normals[part_mask == 1]
+
+            if display:
+                im.show(part_tex, id='part_tex', waittime=1)
+
+        else:
+            cap.grab()
+
+        i += 1
+
+    # merge textures
+    log.info('Computing median texture...')
+    tex_median = np.nanmedian(tex_agg, axis=2)
+
+    log.info('Inpainting unseen areas...')
+    where = np.max(normal_agg, axis=2) > 0.2
+
+    tex_mask = iso.iso_mask
+    mask_final = np.float32(where)
+
+    kernel_size = np.int(resolution * 0.02)
+    kernel = np.ones((kernel_size, kernel_size), np.uint8)
+    inpaint_area = cv2.dilate(tex_mask, kernel) - mask_final
+
+    tex_final = cv2.inpaint(np.uint8(tex_median * 255), np.uint8(inpaint_area * 255), 3, cv2.INPAINT_TELEA)
+
+    cv2.imwrite(out, tex_final)
+    log.info('Done.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        'consensus',
+        type=str,
+        help="pkl file that contains consensus")
+    parser.add_argument(
+        'camera',
+        type=str,
+        help="pkl file that contains camera settings")
+    parser.add_argument(
+        'video',
+        type=str,
+        help="Input video")
+    parser.add_argument(
+        'pose_file',
+        type=str,
+        help="File that contains poses")
+    parser.add_argument(
+        'masks_file',
+        type=str,
+        help="File that contains segmentations")
+    parser.add_argument(
+        'out',
+        type=str,
+        help="Out file path")
+    parser.add_argument(
+        '--model', '-m',
+        default='vendor/smpl/models/basicmodel_m_lbs_10_207_0_v1.0.0.pkl',
+        help='Path to SMPL model')
+    parser.add_argument(
+        '--resolution', '-r', default=1000, type=int,
+        help="Output resolution")
+    parser.add_argument(
+        '--num', '-n', default=120, type=int,
+        help="Number of used frames")
+    parser.add_argument(
+        '--first_frame', '-f', default=0, type=int,
+        help="First frame to use")
+    parser.add_argument(
+        '--last_frame', '-l', default=2000, type=int,
+        help="Last frame to use")
+    parser.add_argument(
+        '--display', '-d',
+        action='store_true',
+        help="Enable visualization")
+
+    args = parser.parse_args()
+
+    main(args.consensus, args.camera, args.video, args.pose_file, args.masks_file, args.out, args.model,
+         args.resolution, args.num, args.first_frame, args.last_frame, args.display)

videoavatars/tex/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-

videoavatars/tex/iso.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import numpy as np
+
+from render.renderer import OrthoTexturedRenderer, OrthoColoredRenderer
+from render.camera import OrthoProjectPoints
+
+
+class Isomapper():
+    def __init__(self, vt, ft, f, tex_res, bgcolor=np.zeros(3)):
+        vt3d = np.dstack((vt[:, 0] - 0.5, 1 - vt[:, 1] - 0.5, np.zeros(vt.shape[0])))[0]
+        ortho = OrthoProjectPoints(rt=np.zeros(3), t=np.zeros(3), near=-1, far=1, left=-0.5, right=0.5, bottom=-0.5,
+                                   top=0.5, width=tex_res, height=tex_res)
+        self.tex_res = tex_res
+        self.f = ft
+        self.ft = f
+        self.rn_tex = OrthoTexturedRenderer(v=vt3d, f=ft, ortho=ortho, vc=np.ones_like(vt3d), bgcolor=bgcolor)
+        self.rn_vis = OrthoColoredRenderer(v=vt3d, f=ft, ortho=ortho, vc=np.ones_like(vt3d), bgcolor=np.zeros(3),
+                                           num_channels=1)
+        self.bgcolor = bgcolor
+        self.iso_mask = np.array(self.rn_vis.r)
+
+    def render(self, frame, proj_v, visible_faces=None):
+        h, w, _ = np.atleast_3d(frame).shape
+        v2d = proj_v.r
+        v2d_as_vt = np.dstack((v2d[:, 0] / w, 1 - v2d[:, 1] / h))[0]
+
+        self.rn_tex.set(texture_image=frame, vt=v2d_as_vt, ft=self.ft)
+        tex = np.array(self.rn_tex.r)
+
+        if visible_faces is not None:
+            self.rn_vis.set(f=self.f[visible_faces])
+            mask = np.atleast_3d(self.rn_vis.r)
+            tex = mask * tex + (1 - mask) * self.bgcolor
+
+        return tex
+
+
+class IsoColoredRenderer:
+    def __init__(self, vt, ft, f, tex_res):
+        ortho = OrthoProjectPoints(rt=np.zeros(3), t=np.zeros(3), near=-1, far=1, left=-0.5, right=0.5, bottom=-0.5,
+                                   top=0.5, width=tex_res, height=tex_res)
+        vt3d = np.dstack((vt[:, 0] - 0.5, 1 - vt[:, 1] - 0.5, np.zeros(vt.shape[0])))[0]
+        vt3d = vt3d[ft].reshape(-1, 3)
+        self.f = f
+        self.rn = OrthoColoredRenderer(bgcolor=np.zeros(3), ortho=ortho, v=vt3d, f=np.arange(ft.size).reshape(-1, 3))
+
+    def render(self, vc):
+        vc = np.atleast_3d(vc)
+
+        if vc.shape[2] == 1:
+            vc = np.hstack((vc, vc, vc))
+
+        self.rn.set(vc=vc[self.f].reshape(-1, 3))
+
+        return np.array(self.rn.r)

videoavatars/util/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-

videoavatars/util/im.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+_figures = {}
+
+
+def show(im, waittime=0, id='plt', max_width=600):
+    plt.ion()
+    w = min(im.shape[1], max_width)
+    h = max_width * (1.0 * im.shape[0]) / im.shape[1] if w == max_width else im.shape[0]
+    plt.figure(id, figsize=(w / 80, h / 80), dpi=80)
+
+    ax = plt.axes([0, 0, 1, 1], frameon=False)
+    ax.get_xaxis().set_visible(False)
+    ax.get_yaxis().set_visible(False)
+
+    if np.issubdtype(im.dtype, np.floating):
+        if np.max(im) > 1:
+            factor = 255 / np.max(im)
+        else:
+            factor = 255
+    else:
+        factor = 1
+
+    if np.atleast_3d(im).shape[2] == 3:
+        data = np.uint8(im * factor)[:, :, ::-1]
+    else:
+        data = np.uint8(np.dstack((im, im, im)) * factor)
+
+    if id in _figures and plt.fignum_exists(id):
+        _figures[id].set_array(data)
+    else:
+        _figures[id] = plt.imshow(data)
+
+    if waittime == 0:
+        plt.waitforbuttonpress()
+    else:
+        plt.pause(waittime / 1000.)
+    plt.ioff()

videoavatars/util/logger.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import logging
+
+__all__ = ['log']
+
+
+FORMAT = "%(asctime)s [%(filename)s:%(funcName)s:%(lineno)d] %(message)s"
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO, format=FORMAT)

videoavatars/util/mesh.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import os
+import numpy as np
+
+
+def write(filename, v, f, vt=None, ft=None, vn=None, vc=None, texture=None):
+    with open(filename, 'w') as fp:
+        if texture is not None:
+            mat_file = filename.replace('obj', 'mtl')
+
+            fp.write('mtllib {}\n'.format(os.path.basename(mat_file)))
+            fp.write('usemtl mat\n')
+
+            with open(mat_file, 'w') as mfp:
+                mfp.write('newmtl mat\n')
+                mfp.write('Ka 1.0 1.0 1.0\n')
+                mfp.write('Kd 1.0 1.0 1.0\n')
+                mfp.write('Ks 0.0 0.0 0.0\n')
+                mfp.write('d 1.0\n')
+                mfp.write('Ns 0.0\n')
+                mfp.write('illum 0\n')
+                mfp.write('map_Kd {}\n'.format(texture))
+
+        if vc is not None:
+            fp.write(('v {:f} {:f} {:f} {:f} {:f} {:f}\n' * len(v)).format(*np.hstack((v, vc)).reshape(-1)))
+        else:
+            fp.write(('v {:f} {:f} {:f}\n' * len(v)).format(*v.reshape(-1)))
+
+        if vn is not None:
+            fp.write(('vn {:f} {:f} {:f}\n' * len(vn)).format(*vn.reshape(-1)))
+
+        if vt is not None:
+            fp.write(('vt {:f} {:f}\n' * len(vt)).format(*vt.reshape(-1)))
+
+        if ft is not None:
+            fp.write(('f {:d}/{:d}/{:d} {:d}/{:d}/{:d} {:d}/{:d}/{:d}\n' * len(f)).format(*np.hstack((f.reshape(-1, 1), ft.reshape(-1, 1), f.reshape(-1, 1))).reshape(-1) + 1))
+        else:
+            fp.write(('f {:d}//{:d} {:d}//{:d} {:d}//{:d}\n' * len(f)).format(*np.repeat(f.reshape(-1) + 1, 2)))