boost lgamma numeric overflow in inference.minimize_nested_blockmodel_dl?

Dear all,

Thanks and apologies in advance if this is something silly on my part. I can blockmodel this dataset using a vector of covariates (two int32’s) without hierarchy no problem, but when I try to add hierarchy, I get an error reproducible on two different platforms (OS X and ubuntu, one without OpenMP support, one with; one from within an ipython notebook, one from without; both Python 2.7.12):

RuntimeError Traceback (most recent call last)
<ipython-input-441-ae6b18f3b139> in <module>()
      1 # infer hierarchical blockmodel with edge covariates
----> 2 get_ipython().magic(u'time hbm_edgecovariates = gt.inference.minimize_nested_blockmodel_dl(g, deg_corr=False, layers=True, state_args=dict(ec=g.ep.weight, layers=False))')

//anaconda/envs/geopandas_test_env/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in magic(self, arg_s)
   2156 magic_name, _, magic_arg_s = arg_s.partition(' ')
   2157 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2158 return self.run_line_magic(magic_name, magic_arg_s)
   2159
   2160 #-------------------------------------------------------------------------

//anaconda/envs/geopandas_test_env/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in run_line_magic(self, magic_name, line)
   2077 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2078 with self.builtin_trap:
-> 2079 result = fn(*args,**kwargs)
   2080 return result
   2081

<decorator-gen-59> in time(self, line, cell, local_ns)

//anaconda/envs/geopandas_test_env/lib/python2.7/site-packages/IPython/core/magic.pyc in <lambda>(f, *a, **k)
    186 # but it's overkill for just that one bit of state.
    187 def magic_deco(arg):
--> 188 call = lambda f, *a, **k: f(*a, **k)
    189
    190 if callable(arg):

//anaconda/envs/geopandas_test_env/lib/python2.7/site-packages/IPython/core/magics/execution.pyc in time(self, line, cell, local_ns)
   1178 else:
   1179 st = clock2()
-> 1180 exec(code, glob, local_ns)
   1181 end = clock2()
   1182 out = None

<timed exec> in <module>()

/usr/local/lib/python2.7/site-packages/graph_tool/inference/minimize.pyc in minimize_nested_blockmodel_dl(g, B_min, B_max, b_min, b_max, Bs, bs, deg_corr, overlap, nonoverlap_init, layers, hierarchy_minimize_args, state_args, bisection_args, mcmc_args, anneal_args, mcmc_equilibrate_args, shrink_args, mcmc_multilevel_args, verbose)
    513 verbose=verbose,
    514 **dmask(hierarchy_minimize_args,
--> 515 ["B_max", "B_min", "bisection_args", "verbose"]))
    516
    517 return state

/usr/local/lib/python2.7/site-packages/graph_tool/inference/nested_blockmodel.pyc in hierarchy_minimize(state, B_min, B_max, b_min, b_max, frozen_levels, sparse_thres, bisection_args, verbose)
    749 bstate = state.find_new_level(l, bisection_args=bisection_args,
    750 B_min=B_min, B_max=B_max,
--> 751 b_min=b_min, b_max=b_max)
    752 else:
    753 bstate = state.find_new_level(l, bisection_args=bisection_args)

/usr/local/lib/python2.7/site-packages/graph_tool/inference/nested_blockmodel.pyc in find_new_level(self, l, sparse_thres, bisection_args, B_min, B_max, b_min, b_max)
    482
    483 # find new state
--> 484 state = bisection_minimize([min_state, max_state], **bisection_args)
    485
    486 if _bm_test():

/usr/local/lib/python2.7/site-packages/graph_tool/inference/bisection.pyc in bisection_minimize(init_states, random_bisection, mcmc_multilevel_args, extra_entropy_args, verbose)
    139 for state in init_states:
    140 b_cache[state.B] = (get_ent(state, mcmc_multilevel_args,
--> 141 extra_entropy_args),
    142 state.copy())
    143

/usr/local/lib/python2.7/site-packages/graph_tool/inference/bisection.pyc in get_ent(state, mcmc_multilevel_args, extra_entropy_args)
     68 mcmc_args = mcmc_equilibrate_args.get("mcmc_args", {})
     69 entropy_args = mcmc_args.get("entropy_args", {})
---> 70 S = state.entropy(**dict(entropy_args, **extra_entropy_args))
     71 return S
     72

/usr/local/lib/python2.7/site-packages/graph_tool/inference/layered_blockmodel.pyc in entropy(self, adjacency, dl, partition_dl, degree_dl, degree_dl_kind, edges_dl, dense, multigraph, deg_entropy, exact, **kwargs)
    616 dense=dense, multigraph=multigraph,
    617 deg_entropy=deg_entropy, exact=exact,
--> 618 **dict(kwargs, test=False))
    619
    620 if dl and edges_dl:

/usr/local/lib/python2.7/site-packages/graph_tool/inference/blockmodel.pyc in entropy(self, adjacency, dl, partition_dl, degree_dl, degree_dl_kind, edges_dl, dense, multigraph, deg_entropy, exact, **kwargs)
    845 callback = kwargs.pop("callback", None)
    846 if callback is not None:
--> 847 S += callback(self)
    848
    849 if kwargs.pop("test", True) and _bm_test():

/usr/local/lib/python2.7/site-packages/graph_tool/inference/nested_blockmodel.pyc in <lambda>(s)
    431 callback=lambda s: get_edges_dl(s,
    432 self.hstate_args,
--> 433 self.hentropy_args))
    434 mcmc_args = dict(mcmc_args, entropy_args=entropy_args,
    435 disable_callback_test=isinstance(self.levels[0],

/usr/local/lib/python2.7/site-packages/graph_tool/inference/nested_blockmodel.pyc in get_edges_dl(state, hstate_args, hentropy_args)
     38 bstate = state.get_block_state(b=bclabel, **hstate_args)
     39 return bstate.entropy(**dict(hentropy_args, dl=True, edges_dl=False,
---> 40 multigraph=True))
     41
     42

/usr/local/lib/python2.7/site-packages/graph_tool/inference/layered_blockmodel.pyc in entropy(self, adjacency, dl, partition_dl, degree_dl, degree_dl_kind, edges_dl, dense, multigraph, deg_entropy, exact, **kwargs)
    628
    629 if dl:
--> 630 S += self.__get_layer_entropy()
    631
    632 if _bm_test() and kwargs.get("test", True):

/usr/local/lib/python2.7/site-packages/graph_tool/inference/layered_blockmodel.pyc in __get_layer_entropy(self)
    590 be = group_vector_property([ec, ec])
    591 lstate = OverlapBlockState(g, b=be, deg_corr=False)
--> 592 self.__layer_entropy = lstate.entropy(dl=True, edges_dl=False) - \
    593 lstate.entropy(dl=False)
    594 else:

/usr/local/lib/python2.7/site-packages/graph_tool/inference/overlap_blockmodel.pyc in entropy(self, adjacency, dl, partition_dl, degree_dl, degree_dl_kind, edges_dl, dense, multigraph, deg_entropy, exact, **kwargs)
    581 multigraph=multigraph,
    582 deg_entropy=deg_entropy, exact=exact,
--> 583 **kwargs)
    584
    585

/usr/local/lib/python2.7/site-packages/graph_tool/inference/blockmodel.pyc in entropy(self, adjacency, dl, partition_dl, degree_dl, degree_dl_kind, edges_dl, dense, multigraph, deg_entropy, exact, **kwargs)
    824 if dl:
    825 if partition_dl:
--> 826 S += self._state.get_partition_dl()
    827
    828 if edges_dl:

RuntimeError: Error in function boost::math::lgamma<double>(double): numeric overflow

How large are your edge weights?

Hi,

I was trying an int32 vector of length 2 where the elements range between 0 and 6*10^8

I’d be happy to scale them down if the problem is the int32.

Thanks,
Luke

The layered SBM is intended for situations where the number of layers
(distinct categorical edge covariates) is much smaller than the number of
edges in the network, otherwise it is unlikely to give useful results. In
your situation it looks like each layer is going to contain a single edge...

But that does not explain the overflow. However, in order for me to debug
further, I will need a complete, self-contained script (including a sample
network) that reproduces the problem.

Best,
Tiag