RuntimeError: Unsupported TypeMeta in ATen: (please report this error)

[kshitizagarwal-dev]

I am loading the image and resizing it to 512x 512 using the code :

Load image

from PIL import Image

Open the image file

def load_images(path,ext):
image = Image.open(path)

subfolders = ["pngImages", "jpgImages"]
main_folder = "/content/Grounded-Segment-Anything/images/"

os.makedirs(main_folder)
os.makedirs(main_folder+subfolders[0])
os.makedirs(main_folder+subfolders[1])
resized_image = image.resize((512, 512))
if ext == 'jpg' :
  resized_image.save("/content/Grounded-Segment-Anything/images/jpgImages/resized_image.jpg")  # Save the resized image to a file
  local_image_path = "/content/Grounded-Segment-Anything/images/jpgImages/resized_image.jpg"
else:
  resized_image.save("/content/Grounded-Segment-Anything/images/pngImages/resized_image.png")  # Save the resized image to a file
  local_image_path = "/content/Grounded-Segment-Anything/images/pngImages/resized_image.png"

image_source, image = load_image(local_image_path)
return image_source, image

Image.fromarray(image_source)

image_source, image = load_images('/content/drive/MyDrive/rust/12.jpg','jpg')
#/content/drive/MyDrive/rust/Heavy rust 5.png
Image.fromarray(image_source)

def load_model_hf(repo_id, filename, ckpt_config_filename, device='cpu'):
cache_config_file = hf_hub_download(repo_id=repo_id, filename=ckpt_config_filename)

args = SLConfig.fromfile(cache_config_file)
args.device = device
model = build_model(args)

cache_file = hf_hub_download(repo_id=repo_id, filename=filename)
checkpoint = torch.load(cache_file, map_location=device)
log = model.load_state_dict(clean_state_dict(checkpoint['model']), strict=False)
print("Model loaded from {} \n => {}".format(cache_file, log))
_ = model.eval()
import os
print(os.getcwd())
return model

ckpt_repo_id = "ShilongLiu/GroundingDINO"
ckpt_filenmae = "groundingdino_swinb_cogcoor.pth"
ckpt_config_filename = "GroundingDINO_SwinB.cfg.py"

groundingdino_model = load_model_hf(ckpt_repo_id, ckpt_filenmae, ckpt_config_filename, device)

detect object using grounding DINO

def detect(image, text_prompt, model, box_threshold = 0.3, text_threshold = 0.25):

boxes, logits, phrases = predict(
model=model,
image=image,
caption=text_prompt,
box_threshold=box_threshold,
text_threshold=text_threshold
)

print(phrases)
annotated_frame,xyxy = annotate1(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
annotated_frame = annotated_frame[...,::-1] # BGR to RGB
return annotated_frame, boxes, logits,xyxy

annotated_frame, detected_boxes, detected_logits,xyxy= detect(image, text_prompt="rust of all type", model=groundingdino_model)
Image.fromarray(annotated_frame)

Error trace:

FutureWarning: The device argument is deprecated and will be removed in v5 of Transformers.
UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
UserWarning: None of the inputs have requires_grad=True. Gradients will be None

RuntimeError Traceback (most recent call last)
in <cell line: 1>()
----> 1 annotated_frame, detected_boxes, detected_logits,xyxy= detect(image, text_prompt="rust of all type", model=groundingdino_model)
2 Image.fromarray(annotated_frame)

24 frames
in detect(image, text_prompt, model, box_threshold, text_threshold)
2 def detect(image, text_prompt, model, box_threshold = 0.3, text_threshold = 0.25):
3
----> 4 boxes, logits, phrases = predict(
5 model=model,
6 image=image,

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/util/inference.py in predict(model, image, caption, box_threshold, text_threshold, device)
65
66 with torch.no_grad():
---> 67 outputs = model(image[None], captions=[caption])
68
69 prediction_logits = outputs["pred_logits"].cpu().sigmoid()[0] # prediction_logits.shape = (nq, 256)

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1509 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1510 else:
-> 1511 return self._call_impl(*args, **kwargs)
1512
1513 def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1518 or _global_backward_pre_hooks or _global_backward_hooks
1519 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520 return forward_call(*args, **kwargs)
1521
1522 try:

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/models/GroundingDINO/groundingdino.py in forward(self, samples, targets, **kw)
311
312 input_query_bbox = input_query_label = attn_mask = dn_meta = None
--> 313 hs, reference, hs_enc, ref_enc, init_box_proposal = self.transformer(
314 srcs, masks, input_query_bbox, poss, input_query_label, attn_mask, text_dict
315 )

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1509 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1510 else:
-> 1511 return self._call_impl(*args, **kwargs)
1512
1513 def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1518 or _global_backward_pre_hooks or _global_backward_hooks
1519 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520 return forward_call(*args, **kwargs)
1521
1522 try:

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/models/GroundingDINO/transformer.py in forward(self, srcs, masks, refpoint_embed, pos_embeds, tgt, attn_mask, text_dict)
256 # Begin Encoder
257 #########################################################
--> 258 memory, memory_text = self.encoder(
259 src_flatten,
260 pos=lvl_pos_embed_flatten,

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1509 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1510 else:
-> 1511 return self._call_impl(*args, **kwargs)
1512
1513 def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1518 or _global_backward_pre_hooks or _global_backward_hooks
1519 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520 return forward_call(*args, **kwargs)
1521
1522 try:

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/models/GroundingDINO/transformer.py in forward(self, src, pos, spatial_shapes, level_start_index, valid_ratios, key_padding_mask, memory_text, text_attention_mask, pos_text, text_self_attention_masks, position_ids)
575 # main process
576 if self.use_transformer_ckpt:
--> 577 output = checkpoint.checkpoint(
578 layer,
579 output,

/usr/local/lib/python3.10/dist-packages/torch/_compile.py in inner(*args, **kwargs)
22 import torch._dynamo
23
---> 24 return torch._dynamo.disable(fn, recursive)(*args, **kwargs)
25
26 return inner

/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py in _fn(*args, **kwargs)
487 dynamo_config_ctx.enter()
488 try:
--> 489 return fn(*args, **kwargs)
490 finally:
491 set_eval_frame(prior)

/usr/local/lib/python3.10/dist-packages/torch/_dynamo/external_utils.py in inner(*args, **kwargs)
15 @functools.wraps(fn)
16 def inner(*args, **kwargs):
---> 17 return fn(*args, **kwargs)
18
19 return inner

/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py in checkpoint(function, use_reentrant, context_fn, determinism_check, debug, *args, **kwargs)
480 "use_reentrant=False."
481 )
--> 482 return CheckpointFunction.apply(function, preserve, *args)
483 else:
484 gen = _checkpoint_without_reentrant_generator(

/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py in apply(cls, *args, **kwargs)
551 # See NOTE: [functorch vjp and autograd interaction]
552 args = _functorch.utils.unwrap_dead_wrappers(args)
--> 553 return super().apply(*args, **kwargs) # type: ignore[misc]
554
555 if not is_setup_ctx_defined:

/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py in forward(ctx, run_function, preserve_rng_state, *args)
259
260 with torch.no_grad():
--> 261 outputs = run_function(*args)
262 return outputs
263

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1509 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1510 else:
-> 1511 return self._call_impl(*args, **kwargs)
1512
1513 def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1518 or _global_backward_pre_hooks or _global_backward_hooks
1519 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520 return forward_call(*args, **kwargs)
1521
1522 try:

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/models/GroundingDINO/transformer.py in forward(self, src, pos, reference_points, spatial_shapes, level_start_index, key_padding_mask)
784 # self attention
785 # import ipdb; ipdb.set_trace()
--> 786 src2 = self.self_attn(
787 query=self.with_pos_embed(src, pos),
788 reference_points=reference_points,

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1509 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1510 else:
-> 1511 return self._call_impl(*args, **kwargs)
1512
1513 def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1518 or _global_backward_pre_hooks or _global_backward_hooks
1519 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520 return forward_call(*args, **kwargs)
1521
1522 try:

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/models/GroundingDINO/ms_deform_attn.py in forward(self, query, key, value, query_pos, key_padding_mask, reference_points, spatial_shapes, level_start_index, **kwargs)
336 attention_weights = attention_weights.float()
337
--> 338 output = MultiScaleDeformableAttnFunction.apply(
339 value,
340 spatial_shapes,

/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py in apply(cls, *args, **kwargs)
551 # See NOTE: [functorch vjp and autograd interaction]
552 args = _functorch.utils.unwrap_dead_wrappers(args)
--> 553 return super().apply(*args, **kwargs) # type: ignore[misc]
554
555 if not is_setup_ctx_defined:

/content/Grounded-Segment-Anything/GroundingDINO/groundingdino/models/GroundingDINO/ms_deform_attn.py in forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step)
51 ):
52 ctx.im2col_step = im2col_step
---> 53 output = _C.ms_deform_attn_forward(
54 value,
55 value_spatial_shapes,

RuntimeError: Unsupported TypeMeta in ATen: (please report this error)

[SubicLovePython]

me too