instancer.h

//
// Copyright 2023 Pixar
//
// Licensed under the terms set forth in the LICENSE.txt file available at
// https://openusd.org/license.
//
#ifndef EXT_RMANPKG_PLUGIN_RENDERMAN_PLUGIN_HD_PRMAN_INSTANCER_H
#define EXT_RMANPKG_PLUGIN_RENDERMAN_PLUGIN_HD_PRMAN_INSTANCER_H
 
#include "hdPrman/concurrentMap.h"
#include "hdPrman/renderParam.h"
#include "hdPrman/rixStrings.h"
 
#include "pxr/imaging/hd/instancer.h"
#include "pxr/imaging/hd/sceneDelegate.h"
#include "pxr/imaging/hd/timeSampleArray.h"
#include "pxr/imaging/hd/types.h"
 
#include "pxr/usd/sdf/path.h"
 
#include "pxr/base/gf/matrix4d.h"
#include "pxr/base/tf/hash.h"
#include "pxr/base/tf/hashmap.h"
#include "pxr/base/tf/token.h"
#include "pxr/base/vt/types.h"
#include "pxr/base/vt/value.h"
 
#include "pxr/pxr.h"
 
#include <Riley.h>
#include <RileyIds.h>
#include <RiTypesHelper.h>
 
#include <tbb/queuing_mutex.h>
#include <tbb/spin_mutex.h>
 
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <initializer_list>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
#define HDPRMAN_INSTANCER_MAX_TIME_SAMPLES 1
 
PXR_NAMESPACE_OPEN_SCOPE
 
class HdPrmanInstancer : public HdInstancer
{
 
public:
 
    HdPrmanInstancer(HdSceneDelegate* delegate, SdfPath const& id);
 
    ~HdPrmanInstancer();
 
    void Sync(HdSceneDelegate *sceneDelegate,
              HdRenderParam   *renderParam,
              HdDirtyBits     *dirtyBits) override;
 
    void Finalize(HdRenderParam *renderParam) override;
 
    HdDirtyBits GetInitialDirtyBitsMask() const override;
 
    void Populate(
        HdRenderParam* renderParam,
        const HdDirtyBits& dirtyBits,
        const SdfPath& hydraPrototypeId,
        const std::vector<riley::GeometryPrototypeId>& rileyPrototypeIds,
        const riley::CoordinateSystemList& coordSysList,
        const RtParamList& prototypeParams,
        const HdTimeSampleArray<GfMatrix4d, HDPRMAN_MAX_TIME_SAMPLES>&
            prototypeXform,
        const std::vector<riley::MaterialId>& rileyMaterialIds,
        const SdfPathVector& prototypePaths,
        const riley::LightShaderId& lightShaderId = riley::LightShaderId::InvalidId());
 
    void Depopulate(
        HdRenderParam* renderParam,
        const SdfPath& prototypePrimPath,
        const std::vector<riley::GeometryPrototypeId>& protoIdsToKeep = {});
 
private:
 
    // **********************************************
    // **              Private Types               **
    // **********************************************
 
    using _GfMatrixSA =
        HdTimeSampleArray<GfMatrix4d, HDPRMAN_INSTANCER_MAX_TIME_SAMPLES>;
    using _RtMatrixSA =
        HdTimeSampleArray<RtMatrix4x4, HDPRMAN_INSTANCER_MAX_TIME_SAMPLES>;
 
    struct _RtParamListHashFunctor
    {
        size_t operator()(const RtParamList& params) const noexcept
        {
            // Wow this sucks, but RtParamList::Hash() is not const!
            RtParamList copy = params;
            return TfHash()(copy.Hash());
        }
    };
 
    struct _RtParamListEqualToFunctor
    {
        bool operator()(const RtParamList& lhs, const RtParamList& rhs) const noexcept
        {
            return _RtParamListHashFunctor()(lhs) == _RtParamListHashFunctor()(rhs);
        }
    };
 
    struct _PrimvarValue
    {
        HdPrimvarDescriptor desc;
        VtValue value;
    };
 
    struct _FlattenData
    {
 
        // The set of light linking categories
        std::unordered_set<TfToken, TfToken::HashFunctor> categories;
 
        // We store visibility in an RtParamList to take advantage of that
        // structure's Inherit and Update methods, and because simply storing
        // a single boolean would clobber any renderer-specific params that might
        // have been authored on a given (native) instance.
        RtParamList params;
 
        // Rendertag-based visibility uses grouping:membership, which is not
        // supported inside prototype groups.
        TfToken renderTag;
 
        _FlattenData() = default;
        _FlattenData(const VtTokenArray& cats)
          : categories(cats.begin(), cats.end()) { }
        _FlattenData(const VtTokenArray& cats, bool vis)
          : categories(cats.begin(), cats.end())
        {
            SetVisibility(vis);
        }
        // Copy constructor
        _FlattenData(const _FlattenData& other)
          : categories(other.categories.cbegin(), other.categories.cend())
          , renderTag(other.renderTag)
        {
            params.Update(other.params);
        }
 
        // Params that already exist here will not be changed;
        // categories will be merged; renderTag will be updated only
        // if it was empty.
        void Inherit(const _FlattenData& rhs)
        {
            categories.insert(rhs.categories.cbegin(), rhs.categories.cend());
            params.Inherit(rhs.params);
            if (renderTag.IsEmpty()) {
                renderTag = rhs.renderTag;
            }
        }
 
        // Params that already exist here will be changed;
        // categories will be merged; renderTag will be overwritten unless
        // empty on rhs.
        void Update(const _FlattenData& rhs)
        {
            categories.insert(rhs.categories.cbegin(), rhs.categories.cend());
            params.Update(rhs.params);
            if (!rhs.renderTag.IsEmpty()) {
                renderTag = rhs.renderTag;
            }
        }
 
        // Update this FlattenData's visibility from an RtParamList. Visibility
        // params that already exist here will be changed; visibility and
        // light linking params on the RtParamList will be removed from it.
        void UpdateVisAndFilterParamList(RtParamList& other) {
            // Move visibility params from the RtParamList to the FlattenData
            for (const RtUString& param : _GetVisibilityParams()) {
                int val;
                if (other.GetInteger(param, val)) {
                    if (val == 1) {
                        params.Remove(param);
                    } else {
                        params.SetInteger(param, val);
                    }
                    other.Remove(param);
                }
            }
 
            // Copy any existing value for grouping:membership into the
            // flatten data. For lights, this gets a value during light sync,
            // and ConvertCategoriesToAttributes specifically handles preserving
            // it. We need to capture the value from light sync here so we can
            // and flatten against it. It won't be captured by the categories
            // because the value set in light sync comes from a different
            // source. It has to be handled separately from categories.
            RtUString groupingMembership;
            if (other.GetString(RixStr.k_grouping_membership, groupingMembership)) {
                params.SetString(RixStr.k_grouping_membership, groupingMembership);
            }
 
            // Remove the light linking params from the RtParamList. Not going
            // to parse them back out to individual tokens to add to
            // the FlattenData categories, as they will be captured elsewhere.
            for (const RtUString& param : _GetLightLinkParams()) {
                other.Remove(param);
            }
        }
 
        // Sets all visibility params, overwriting current values.)
        void SetVisibility(bool visible) {
            if (visible) {
                for (const RtUString& param : _GetVisibilityParams()) {
                    params.Remove(param);
                }
            } else {
                for (const RtUString& param : _GetVisibilityParams()) {
                    params.SetInteger(param, 0);
                }
            }
        }
 
        // equals operator
        bool operator==(const _FlattenData& rhs) const noexcept
        {
            return categories == rhs.categories &&
                _RtParamListEqualToFunctor()(params, rhs.params) &&
                renderTag == rhs.renderTag;
        }
 
        struct HashFunctor {
            size_t operator()(const _FlattenData& fd) const noexcept
            {
                size_t hash = 0ul;
 
                // simple order-independent XOR hash aggregation
                for (const TfToken& tok : fd.categories) {
                    hash ^= TfHash()(tok.GetString());
                }
                hash ^= _RtParamListHashFunctor()(fd.params);
                hash ^= TfHash()(fd.renderTag.GetString());
                return hash;
            }
        };
 
        std::string ToString() const;
 
    private:
        static std::vector<RtUString> _GetLightLinkParams()
        {
            // List of riley instance params pertaining to light-linking that are
            // not supported on instances inside geometry prototype groups
            static const std::vector<RtUString> LightLinkParams = {
                RixStr.k_lightfilter_subset,
                RixStr.k_lighting_subset,
                RixStr.k_grouping_membership,
                RixStr.k_lighting_excludesubset
            };
            return LightLinkParams;
        }
 
        static std::vector<RtUString> _GetVisibilityParams()
        {
            // List of rile instance params pertaining to visibility that are
            // not supported on instances inside geometry prototype groups
            static const std::vector<RtUString> VisParams = {
                RixStr.k_visibility_camera,
                RixStr.k_visibility_indirect,
                RixStr.k_visibility_transmission
            };
            return VisParams;
        }
 
    };
 
    struct _InstanceData
    {
        _FlattenData flattenData;
        RtParamList params;
        _GfMatrixSA transform;
 
        _InstanceData() = default;
        _InstanceData(const RtParamList& p, const _GfMatrixSA& xform)
            : transform(xform)
        {
            params.Inherit(p);
        }
    };
 
    using _FlattenGroupMap = HdPrmanConcurrentMap<
        _FlattenData,
        riley::GeometryPrototypeId,
        _FlattenData::HashFunctor>;
 
    struct _RileyInstanceId
    {
        riley::GeometryPrototypeId groupId;
        riley::GeometryInstanceId geoInstanceId;
        riley::LightInstanceId lightInstanceId;
    };
 
    using _InstanceIdVec = std::vector<_RileyInstanceId>;
 
    struct _ProtoIdHash
    {
        size_t operator()(const riley::GeometryPrototypeId& id) const noexcept
        {
            return std::hash<uint32_t>()(id.AsUInt32());
        }
    };
 
    using _ProtoInstMap = std::unordered_map<
        riley::GeometryPrototypeId,
        _InstanceIdVec,
        _ProtoIdHash>;
 
    using _ProtoGroupCounterMap = HdPrmanConcurrentMap<
        riley::GeometryPrototypeId,
        std::atomic<int>,
        _ProtoIdHash>;
 
    struct _ProtoMapEntry
    {
        _ProtoInstMap map;
        bool dirty;
    };
 
    struct _ChildPopulateLockMapKey
    {
        SdfPath childInstancerId;
        SdfPath prototypePrimPath;
 
        struct Hash {
            size_t operator()(const _ChildPopulateLockMapKey& key) const
            {
                return TfHash::Combine(
                    key.childInstancerId, key.prototypePrimPath);
            }
        };
 
        bool operator==(const _ChildPopulateLockMapKey& other) const
        {
            return childInstancerId == other.childInstancerId &&
                prototypePrimPath == other.prototypePrimPath;
        }
    };
 
    using _ProtoMap = HdPrmanConcurrentMap<
        SdfPath, _ProtoMapEntry, SdfPath::Hash>;
 
    using _ChildPopulateLockMap = HdPrmanConcurrentMap<
        _ChildPopulateLockMapKey,
        tbb::queuing_mutex,
        _ChildPopulateLockMapKey::Hash>;
 
    // **********************************************
    // **             Private Methods              **
    // **********************************************
 
    // Sync helper; caches instance-rate primvars
    void _SyncPrimvars(const HdDirtyBits* dirtyBits);
 
    // Sync helper; caches the instancer and instance transforms
    void _SyncTransforms(const HdDirtyBits* dirtyBits, HdPrman_RenderParam *);
 
    // Sync helper; caches instance or instancer categories as appropriate
    void _SyncCategories(const HdDirtyBits* dirtyBits);
 
    // Sync helper; caches instancer visibility
    void _SyncVisibility(const HdDirtyBits* dirtyBits);
 
    // Sync helper; caches instancer render tag
    void _SyncRenderTag(const HdDirtyBits* dirtyBits);
 
    // Generates InstanceData structures for this instancer's instances;
    // will multiply those by any supplied subInstances
    void _ComposeInstances(
        const SdfPath& protoId,
        const std::vector<_InstanceData>& subInstances,
        std::vector<_InstanceData>& instances);
 
    // Generates FlattenData from a set of instance params by looking for
    // incompatible params and moving them from the RtParamList to the
    // FlattenData. Called by _ComposeInstances().
    void _ComposeInstanceFlattenData(
        size_t instanceId,
        RtParamList& instanceParams,
        _FlattenData& fd,
        const _FlattenData& fromBelow = _FlattenData());
 
    // Generates param sets and flatten data for the given prototype
    // prim(s). Starts with copies of the prototype params provided to
    // Populate, and additionally captures constant/uniform params inherited
    // by the prototype, prototype- and subset-level light linking, and subset
    // visibility.
    void _ComposePrototypeData(
        const SdfPath& protoPath,
        const RtParamList& globalProtoParams,
        bool isLight,
        const std::vector<riley::GeometryPrototypeId>& protoIds,
        const SdfPathVector& subProtoPaths,
        const std::vector<_FlattenData>& subProtoFlats,
        std::vector<RtParamList>& protoParams,
        std::vector<_FlattenData>& protoFlats);
 
    // Gathers knownProtoIds from protoMapEntry, then:
    //     newProtoIds  = (inputProtoIds - knownProtoIds)
    //     deadProtoIds = (knownProtoIds - inputProtoIds)
    // Returns true unless both output sets are empty.
    static
    bool _GatherChangedPrototypeIds(
        const _ProtoMapEntry& protoMapEntry,
        const std::vector<riley::GeometryPrototypeId>& inputProtoIds,
        std::vector<riley::GeometryPrototypeId>& newProtoIds,
        std::vector<riley::GeometryPrototypeId>& deadProtoIds);
 
    // Flags all previously seen prototype prim paths as needing their instances
    // updated the next time they show up in a Populate call.
    void _SetPrototypesDirty();
 
    // Generates instances of the given prototypes according to the instancer's
    // instancing configuration. If the instancer is too deep, they get passed
    // up to this method on the parent instancer. The given prototypes may be
    // prims (when called through the public Populate method) or may be
    // child instancers represented by riley geometry prototype groups. In
    // either case, the caller owns the riley prototypes. _PopulateInstances()
    // is thread-safe except when called for the same hydraPrototypeId and
    // prototypePrimPath. That's only possible from child instancers, so those
    // must use _PopulateInstancesForChild() instead.
    void _PopulateInstances(
        HdRenderParam* renderParam,
        const HdDirtyBits& dirtyBits,
        const SdfPath& hydraPrototypeId,
        const SdfPath& prototypePrimPath,
        const std::vector<riley::GeometryPrototypeId>& rileyPrototypeIds,
        const riley::CoordinateSystemList& coordSysList,
        const RtParamList& prototypeParams,
        const HdTimeSampleArray<GfMatrix4d, HDPRMAN_INSTANCER_MAX_TIME_SAMPLES>&
            prototypeXform,
        const std::vector<riley::MaterialId>& rileyMaterialIds,
        const SdfPathVector& prototypePaths,
        const riley::LightShaderId& lightShaderId,
        const std::vector<_InstanceData>& subInstances,
        const std::vector<_FlattenData>& prototypeFlats);
 
    // Locks before calling _PopulateInstances() to prevent duplicated Riley
    // calls that may arise when Populate() has been called from multiple
    // threads from the same child instancer. Locks are segregated by
    // childInstancerId and prototypePrimPath to avoid over-locking.
    void _PopulateInstancesForChild(
        HdRenderParam* renderParam,
        const HdDirtyBits& dirtyBits,
        const SdfPath& childInstancerId,
        const SdfPath& prototypePrimPath,
        const std::vector<riley::GeometryPrototypeId>& rileyPrototypeIds,
        const riley::CoordinateSystemList& coordSysList,
        const RtParamList& prototypeParams,
        const HdTimeSampleArray<GfMatrix4d, HDPRMAN_INSTANCER_MAX_TIME_SAMPLES>&
            prototypeXform,
        const std::vector<riley::MaterialId>& rileyMaterialIds,
        const SdfPathVector& prototypePaths,
        const riley::LightShaderId& lightShaderId,
        const std::vector<_InstanceData>& subInstances,
        const std::vector<_FlattenData>& prototypeFlats);
 
    void _DepopulateInstances(
        HdRenderParam* renderParam,
        const SdfPath& prototypePrimPath,
        const std::vector<riley::GeometryPrototypeId>& protoIdsToKeep = {});
 
    // Locks before calling _DepopulateInstances(), using the same locking
    // strategy as _PopulateInstancesForChild().
    void _DepopulateInstancesForChild(
        HdRenderParam* renderParam,
        const SdfPath& childInstancerId,
        const SdfPath& prototypePrimPath,
        const std::vector<riley::GeometryPrototypeId>& protoIdsToKeep = {});
 
    // Get pointer to parent instancer, if one exists
    HdPrmanInstancer* _GetParentInstancer();
 
    // Resize the instancer's interal state store for tracking riley instances.
    // Shrinking the number of instances for a given prototype path and id will
    // delete excess instances from riley. Call with newSize = 0 to kill 'em all.
    void _ResizeProtoMap(
        riley::Riley* riley,
        const SdfPath& prototypePrimPath,
        const std::vector<riley::GeometryPrototypeId>& rileyPrototypeIds,
        size_t newSize);
 
    // Deletes any riley geometry prototype groups that are no longer needed.
    // Returns true if any groups were deleted.
    bool _CleanDisusedGroupIds(HdPrman_RenderParam* param);
 
    // Obtain the riley geometry prototype group id for a given FlattenData.
    // Returns true if the group had to be created. Gives InvalidId when this
    // instancer has no parent instancer. Also atomically increments the group's
    // instance counter, under the assumption that a new instance will be added
    // to the group. This is necessary to prevent possible deletion of a newly-
    // created prototype group by another thread. If a new instance is not added
    // to the group, you must decrement the counter afterwards.
    bool _AcquireGroupIdAndIncrementCounter(
        HdPrman_RenderParam* param,
        const _FlattenData& flattenGroup,
        riley::GeometryPrototypeId& groupId);
 
    // Retrieves instance-rate params for the given instance index from
    // the instancer's cache.
    void _GetInstanceParams(
        size_t instanceIndex,
        RtParamList& params);
 
    // Gets constant and uniform params for the prototype
    void _GetPrototypeParams(
        const SdfPath& protoPath,
        RtParamList& params
    );
 
    // Retrieves the instance transform for the given index from the
    // instancer's cache.
    void _GetInstanceTransform(
        size_t instanceIndex,
        _GfMatrixSA& xform,
        const _GfMatrixSA& left = _GfMatrixSA());
 
    // Calculates this instancer's depth in the nested instancing hierarchy.
    // An uninstanced instancer has depth 0. Instancers with depth > 4 cannot
    // use riley nested instancing and must flatten their instances into their
    // parents.
    int _Depth();
 
    // Wrapper around ++(_groupCounters[groupId]) that encapsulates
    // debug messaging. This should only ever be called under lock
    // from _AcquireGroupIdAndIncrementCounter()!
    void _IncrementGroupCounter(const riley::GeometryPrototypeId& id);
 
    // Wrapper around --(_groupCounters[groupId]) that encapsulates
    // debug messaging. This is safe to call at any time.
    void _DecrementGroupCounter(const riley::GeometryPrototypeId& id);
 
    // **********************************************
    // **             Private Members              **
    // **********************************************
 
    // This instancer's cached instance transforms
    HdTimeSampleArray<VtMatrix4dArray, HDPRMAN_INSTANCER_MAX_TIME_SAMPLES> _sa;
 
    // This instancer's cached coordinate system list
    riley::CoordinateSystemList _coordSysList = { 0, nullptr };
 
    // This instancer's cached instance categories; will be empty under point
    // instancing, so all indexing must be bounds-checked!
    std::vector<VtTokenArray> _instanceCategories;
 
    // This instancer's cached visibility and categories
    _FlattenData _instancerFlat;
 
    // This instancer's cached USD primvars
    TfHashMap<TfToken, _PrimvarValue, TfToken::HashFunctor> _primvarMap;
 
    // Map of FlattenData to GeometryProtoypeId
    // We use this map to put instances that share values for instance params
    // that are incompatible with riley nesting into shared prototype groups so
    // that the incompatible params may be set on the outermost riley
    // instances of those groups where they are supported. This map may be
    // written to during Populate, so access must be gated behind a mutex
    // lock (built into HdPrmanConcurrentMap).
    _FlattenGroupMap _groupMap;
 
    // Counters for tracking number of instances in each prototype group. Used
    // to speed up empty prototype group removal.
    _ProtoGroupCounterMap _groupCounters;
 
    // riley geometry prototype groups are created during Populate; these must
    // be serialized to prevent creating two different groups for the same set
    // of flatten data.
    tbb::spin_mutex _groupIdAcquisitionLock;
 
    // Main storage for tracking riley instances owned by this instancer.
    // Instance ids are paired with their containing group id (RileyInstanceId),
    // then grouped by their riley geometry prototype id (ProtoInstMap). These
    // are then grouped by id of the prototype prim they represent (which may be
    // the invalid id in the case of analytic lights). The top level of this
    // nested structure may be written to during Populate, therefore access to
    // the top level is gated behind a mutex lock (built into LockingMap).
    // Deeper levels are only ever written to from within a single call to
    // Populate, so they do not have gated access.
    _ProtoMap _protoMap;
 
    // Locks used by _PopulateInstancesFromChild() to serialize parallel calls
    // from the same child instancer for the same prototype prim, which can
    // occur when the child has multiple prototype prims that are being synced
    // in parallel.
    _ChildPopulateLockMap _childPopulateLocks;
};
 
PXR_NAMESPACE_CLOSE_SCOPE
 
#endif  // EXT_RMANPKG_PLUGIN_RENDERMAN_PLUGIN_HD_PRMAN_INSTANCER_H