Solve the problem that module ‘torch.nn.parameter’ has no attribute 'UninitializedParameter’ error occurs when using torch-geometric to build a graph neural network. As shown below
Basic version book information
torch 1.6.0
cuda 10.1
python 3.7.4< a i=4> torch-geometric 2.0.3 torch-geometricdependent version book torch-scatter 2.0.5 torch -sparse 0.6.7
Error reason
You can see that the error is reported in torch, so this is a problem of version mismatch.
In torch version 1.6.0, ‘torch.nn.parameter’ does not have the attribute ‘UninitializedParameter’. After checking the relevant documents, I found that this attribute was added in 1.8.0. So upgrading torch to >=1.8.0 version can solve this problem? (That's the theory, I haven't tried it) Below is my solution.
Solution
This method is actually provided by the torch-geometric official website. Many people mentioned this problem on the official website, so two weeks ago, they gave a solution. [Click here to view], you can see that they have made changes by looking at the source code below (https://github.com/pyg-team/pytorch_geometric/pull/4086/files/b57dc8ec75f6ce8dd1ce399c2a0c3fdab672b3a8…43702e47c89cf0194dfadcc66de2e5cedf796ce1)
To put it simply, the code of the file torch_geometric\nn\dense\linear.py has been slightly modified. The 'UninitializedParameter' attribute of 'torch.nn.parameter' will not be used, so no error will be reported after the modification. As shown below
The changes are like the part of the picture frame I mentioned above. The line pointed to by "-" means you delete it, and you can just add the line pointed to by "+". Then you can change these two things and the problem will be solved. (Of course, directly updating to the latest version should also solve the problem)
Liner.py file
Below is the entire file after I modified it
import copy
import math
from typing import Optional
from typing import Any, Optional
import torch
import torch.nn.functional as F
from torch import Tensor, nn
from torch.nn.parameter import Parameter
from torch_geometric.nn import inits
def is_uninitialized_parameter(x: Any) -> bool:
if not hasattr(nn.parameter, 'UninitializedParameter'):
return False
return isinstance(x, nn.parameter.UninitializedParameter)
class Linear(torch.nn.Module):
r"""Applies a linear tranformation to the incoming data
.. math::
\mathbf{x}^{\prime} = \mathbf{x} \mathbf{W}^{\top} + \mathbf{b}
similar to :class:`torch.nn.Linear`.
It supports lazy initialization and customizable weight and bias
initialization.
Args:
in_channels (int): Size of each input sample. Will be initialized
lazily in case it is given as :obj:`-1`.
out_channels (int): Size of each output sample.
bias (bool, optional): If set to :obj:`False`, the layer will not learn
an additive bias. (default: :obj:`True`)
weight_initializer (str, optional): The initializer for the weight
matrix (:obj:`"glorot"`, :obj:`"uniform"`, :obj:`"kaiming_uniform"`
or :obj:`None`).
If set to :obj:`None`, will match default weight initialization of
:class:`torch.nn.Linear`. (default: :obj:`None`)
bias_initializer (str, optional): The initializer for the bias vector
(:obj:`"zeros"` or :obj:`None`).
If set to :obj:`None`, will match default bias initialization of
:class:`torch.nn.Linear`. (default: :obj:`None`)
Shapes:
- **input:** features :math:`(*, F_{in})`
- **output:** features :math:`(*, F_{out})`
"""
def __init__(self, in_channels: int, out_channels: int, bias: bool = True,
weight_initializer: Optional[str] = None,
bias_initializer: Optional[str] = None):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.weight_initializer = weight_initializer
self.bias_initializer = bias_initializer
if in_channels > 0:
self.weight = Parameter(torch.Tensor(out_channels, in_channels))
else:
self.weight = nn.parameter.UninitializedParameter()
self._hook = self.register_forward_pre_hook(
self.initialize_parameters)
if bias:
self.bias = Parameter(torch.Tensor(out_channels))
else:
self.register_parameter('bias', None)
self._load_hook = self._register_load_state_dict_pre_hook(
self._lazy_load_hook)
self.reset_parameters()
def __deepcopy__(self, memo):
out = Linear(self.in_channels, self.out_channels, self.bias
is not None, self.weight_initializer,
self.bias_initializer)
if self.in_channels > 0:
out.weight = copy.deepcopy(self.weight, memo)
if self.bias is not None:
out.bias = copy.deepcopy(self.bias, memo)
return out
def reset_parameters(self):
if self.in_channels <= 0:
pass
elif self.weight_initializer == 'glorot':
inits.glorot(self.weight)
elif self.weight_initializer == 'uniform':
bound = 1.0 / math.sqrt(self.weight.size(-1))
torch.nn.init.uniform_(self.weight.data, -bound, bound)
elif self.weight_initializer == 'kaiming_uniform':
inits.kaiming_uniform(self.weight, fan=self.in_channels,
a=math.sqrt(5))
elif self.weight_initializer is None:
inits.kaiming_uniform(self.weight, fan=self.in_channels,
a=math.sqrt(5))
else:
raise RuntimeError(f"Linear layer weight initializer "
f"'{self.weight_initializer}' is not supported")
if self.bias is None or self.in_channels <= 0:
pass
elif self.bias_initializer == 'zeros':
inits.zeros(self.bias)
elif self.bias_initializer is None:
inits.uniform(self.in_channels, self.bias)
else:
raise RuntimeError(f"Linear layer bias initializer "
f"'{self.bias_initializer}' is not supported")
def forward(self, x: Tensor) -> Tensor:
r"""
Args:
x (Tensor): The features.
"""
return F.linear(x, self.weight, self.bias)
@torch.no_grad()
def initialize_parameters(self, module, input):
if is_uninitialized_parameter(self.weight):
self.in_channels = input[0].size(-1)
self.weight.materialize((self.out_channels, self.in_channels))
self.reset_parameters()
self._hook.remove()
delattr(self, '_hook')
def _save_to_state_dict(self, destination, prefix, keep_vars):
if is_uninitialized_parameter(self.weight):
destination[prefix + 'weight'] = self.weight
else:
destination[prefix + 'weight'] = self.weight.detach()
if self.bias is not None:
destination[prefix + 'bias'] = self.bias.detach()
def _lazy_load_hook(self, state_dict, prefix, local_metadata, strict,
missing_keys, unexpected_keys, error_msgs):
weight = state_dict[prefix + 'weight']
if is_uninitialized_parameter(weight):
self.in_channels = -1
self.weight = nn.parameter.UninitializedParameter()
if not hasattr(self, '_hook'):
self._hook = self.register_forward_pre_hook(
self.initialize_parameters)
elif is_uninitialized_parameter(self.weight):
self.in_channels = weight.size(-1)
self.weight.materialize((self.out_channels, self.in_channels))
if hasattr(self, '_hook'):
self._hook.remove()
delattr(self, '_hook')
def __repr__(self) -> str:
return (f'{self.__class__.__name__}({self.in_channels}, '
f'{self.out_channels}, bias={self.bias is not None})')
class HeteroLinear(torch.nn.Module):
r"""Applies separate linear tranformations to the incoming data according
to types
.. math::
\mathbf{x}^{\prime}_{\kappa} = \mathbf{x}_{\kappa}
\mathbf{W}^{\top}_{\kappa} + \mathbf{b}_{\kappa}
for type :math:`\kappa`.
It supports lazy initialization and customizable weight and bias
initialization.
Args:
in_channels (int): Size of each input sample. Will be initialized
lazily in case it is given as :obj:`-1`.
out_channels (int): Size of each output sample.
num_types (int): The number of types.
**kwargs (optional): Additional arguments of
:class:`torch_geometric.nn.Linear`.
Shapes:
- **input:**
features :math:`(*, F_{in})`,
type vector :math:`(*)`
- **output:** features :math:`(*, F_{out})`
"""
def __init__(self, in_channels: int, out_channels: int, num_types: int,
**kwargs):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.lins = torch.nn.ModuleList([
Linear(in_channels, out_channels, **kwargs)
for _ in range(num_types)
])
self.reset_parameters()
def reset_parameters(self):
for lin in self.lins:
lin.reset_parameters()
def forward(self, x: Tensor, type_vec: Tensor) -> Tensor:
r"""
Args:
x (Tensor): The input features.
type_vec (LongTensor): A vector that maps each entry to a type.
"""
out = x.new_empty(x.size(0), self.out_channels)
for i, lin in enumerate(self.lins):
mask = type_vec == i
out[mask] = lin(x[mask])
return out
def __repr__(self) -> str:
return (f'{self.__class__.__name__}({self.in_channels}, '
f'{self.out_channels}, bias={self.lins[0].bias is not None})')