{"created":"2023-05-15T15:29:37.126552+00:00","id":19386,"links":{},"metadata":{"_buckets":{"deposit":"adae222d-e686-4055-94f0-174dd69ba0b0"},"_deposit":{"created_by":15,"id":"19386","owners":[15],"pid":{"revision_id":0,"type":"depid","value":"19386"},"status":"published"},"_oai":{"id":"oai:sucra.repo.nii.ac.jp:00019386","sets":["94:429:431:432:992"]},"author_link":[],"item_113_alternative_title_1":{"attribute_name":"タイトル（別言語）","attribute_value_mlt":[{"subitem_alternative_title":"高齢者のための知的追従する買い物支援ロボット"}]},"item_113_biblio_info_9":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicIssueDates":{"bibliographicIssueDate":"2020","bibliographicIssueDateType":"Issued"}}]},"item_113_date_35":{"attribute_name":"作成日","attribute_value_mlt":[{"subitem_date_issued_datetime":"2021-09-16","subitem_date_issued_type":"Created"}]},"item_113_date_granted_20":{"attribute_name":"学位授与年月日","attribute_value_mlt":[{"subitem_dategranted":"2020-09-23"}]},"item_113_degree_grantor_22":{"attribute_name":"学位授与機関","attribute_value_mlt":[{"subitem_degreegrantor":[{"subitem_degreegrantor_name":"埼玉大学"}],"subitem_degreegrantor_identifier":[{"subitem_degreegrantor_identifier_name":"12401","subitem_degreegrantor_identifier_scheme":"kakenhi"}]}]},"item_113_degree_name_21":{"attribute_name":"学位名","attribute_value_mlt":[{"subitem_degreename":"博士(学術)"}]},"item_113_description_13":{"attribute_name":"形態","attribute_value_mlt":[{"subitem_description":"xi, 59p","subitem_description_type":"Other"}]},"item_113_description_23":{"attribute_name":"抄録","attribute_value_mlt":[{"subitem_description":"The lack of caregivers in an aging society is a major social problem. Without assistance, many of the elderly and disabled are unable to perform daily tasks. One important daily activity is shopping in supermarkets. Carrying heavy weighted goods or pushing a shopping cart and moving it from shelf to shelf is tiring and laborious, especially for customers with certain disabilities or the elderly.\n\nMany researcher develop person following robot for supporting elderly in shopping mall but just following may not sufficient or good enough for supporting elderly. Considering the body orientation of the customer it is better to find the appropriate positional relation between the customer and the robot.In addition to the robust person-following, the robot can more support the user if it can act in advance to meet the user's next move. For example, when the user picks up a product from a shelf, it is convenient if the robot automatically comes to the user's right hand side （if the user is right-handed） so that he or she can put it easily in the basket. To realize such functions, the robot needs to recognize the user's behavior.\n\nThe first part of the work is on developing body orientation based shopping support robot. To do that, we address the problem of real-time human pose-based robust person tracking system. We achieve this by cropping the target person's body from the image and then apply a color histogram matching algorithm for tracking a unique person. After tracking the person, we used an omnidirectional camera and LiDAR sensor to find the target person's location and distance from the robot. When the target person stop in front of shopping shelves our robot finds the target person's body movement orientation using our proposed methodology. According to the body orientation our robot assumes a suitable position so that the target person can easily put his shopping product in the basket. Our proposed system was verified in real time environments and it shows that our robot system is highly effective at following a given target person and provides proper support while shopping the target person.\nThe next step was to develop an intelligent shopping support robot that can carry a shopping cart while following its owners and provide the shopping support by observing the customer's head orientation, body orientation and recognizing different shopping behaviors. Recognizing shopping behavior or the intensity of such action is important for understanding the best way to support the customer without disturbing him or her. This system also liberates elderly and disabled people from the burden of pushing shopping carts, because our proposed shopping cart is essentially a type of autonomous mobile robots that recognizes its owner and following him or her. The proposed system discretizes the head and body orientation of customer into 8 directions to estimate whether the customer is looking or turning towards a merchandise shelf. From the robot's video stream, a DNN-based human pose estimator called OpenPose is used to extract the skeleton of 18 joints for each detected body. Using this extracted body joints information, we built a dataset and developed a novel Gated Recurrent Neural Network （GRU） topology to classify different actions that are typically performed in front of shelves: reach to shelf, retract from shelf, hand in shelf, inspect product, inspect shelf. Our GRU network model takes series of 32 frames skeleton data then gives the prediction. Using cross-validation tests, our model achieves an overall accuracy of 82%, which is a significant result. Finally, from the customer's head orientation, body orientation and shopping behavior recognition we develop a complete system for our shopping support robot.\n\nTo operate our robot in a practical environment we must ensure three requirements such as speed, accuracy and cost. OpenPose based model does not ful-fill these requirements. For this reason, we replace OpenPose model with Kinect V2 depth camera. Kinect camera can detect 3D skeleton with approximately 30 frames/sec whereas OpenPose model detect skeleton approximately 5 frames/sec. The accuracy of 3D skeleton based shopping action recognition is high and it is 95% using our GRU network. Using Kinect camera we can measure the distance from robot to tracked person so extra LiDAR sensor does not need. So, Kinect based model is cost effective and does not need extra processing.\n\nFinally, we develop a person following shopping support robot using a Kinect camera that can recognize customer shopping actions or activities. Our robot can follow within a certain distance behind the customer. Whenever our robot detects the customer performing a \"hand in shelf\" action in front of a shelf it positions itself beside the customer with a shopping basket so that the customer can easily put his or her product in the basket. Afterwards, the robot again follows the customer from shelf to shelf until he or she is done with shopping. We conduct our experiments in a real supermarket to evaluate its effectiveness.","subitem_description_type":"Abstract"}]},"item_113_description_24":{"attribute_name":"目次","attribute_value_mlt":[{"subitem_description":"1 Introduction 1\n1.1 Motivation . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 1\n1.2 Objectives . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 2\n1.3 Research Contributions . . . . . . . . . . . . . . . . . . . . . . . . . . 2\n1.4 Organization of Sections . . . . . . . . . . . . . . . . . . . . . . . . . 3\n\n2 Background and Literature Review 5\n2.1 Definition of Human Support Robots . . . . . . . . . . . . . . . . . . 7\n2.2 Potential Application of Human Support Robot . . . . . . . . . . . . 7\n2.2.1 Human Support Robot in Shopping Mall . . . . . . . . . . . . 7\n2.2.2 Human Support Robot in Other Disciplines . . . . . . . . . . 9\n2.3 Customer Shopping Behavior Recognition . . . . . . . . . . . . . . . 10\n2.3.1 Shopping Behavior Understanding . . . . . . . . . . . . . . . . 10\n2.3.2 Shopping Actions . . . . . . . . . . . . . . . . . . . . . . . . . 12\n2.4 Chapter Summery . . . . . . . . . . . . . . . . . . . . . . . . . . . . 15\n\n3 A Person-Following Shopping Support Robot Based on Human Pose Skeleton Data and LiDAR Sensor 17\n3.1 Introduction . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 17\n3.2 Design Approach . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 18\n3.3 Person Tracking . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 18\n3.4 Positional Angle Value of Tracked Person . . . . . . . . . . . . . . . . 19\n3.5 Calculate the Rotation and Translation Value . . . . . . . . . . . . . 20\n3.6 Body Orientation Angle of Target Person . . . . . . . . . . . . . . . . 21\n3.7 Head Orientation Angle of Target Person . . . . . . . . . . . . . . . . 24\n3.8 Experimental Results . . . . . . . . . . . . . . . . . . . . . . . . . . . 24\n3.9 Chapter Summary . . . . . . . . . . . . . . . . . . . . . . . . . . . . 28\n\n4 An Intelligent Shopping Support Robot: Understanding Shopping\nBehavior from 2D Skeleton Data Using GRU Network 29\n4.1 Introduction . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 29\n4.2 Definition of Customer Behavior Model . . . . . . . . . . . . . . . . . 30\n4.3 Framework of Customer Behavior Classification . . . . . . . . . . . . 30\n4.4 Gated Recurrent Neural Network (GRU) . . . . . . . . . . . . . . . . 33\n4.4.1 Dataset Construction . . . . . . . . . . . . . . . . . . . . . . . 34\n4.4.2 Experiments Description . . . . . . . . . . . . . . . . . . . . . 35\n4.5 Architecture of the Shopping Support Robot Based on the User's Behavior Recognition . . . . . . . . . . .  . . . 36\n4.6 Results . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 37\n4.6.1 Evaluation of Behavior Recognition . . . . . . . . . . . . . . . 37\n4.6.1.1 Performance Metrics . . . . . . . . . . . . . . . . . . 37\n4.6.2 Evaluation of Shopping Support Robot . . . . . . . . . . . . . 40\n4.7 Chapter Summary . . . . . . . . . . . . . . . . . . . . . . . . . . . . 41\n\n5 Person-Following Shopping Support Robot using Kinect Depth Camera based on 3D Skeleton Tracking 42\n5.1 Introduction . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 42\n5.2 Design Approach . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 44\n5.2.1 Person's Skeleton Tracking . . . . . . . . . . . . . . . . . . . . 44\n5.2.2 Person Following Procedure of Our Robot . . . . . . . . . . . 45\n5.2.3 Shopping Behavior Action Recognition . . . . . . . . . . . . . 47\n5.2.3.1 Dataset Construction . . . . . . . . . . . . . . . . . . 47\n5.2.3.2 Training the GRU Network . . . . . . . . . . . . . . 47\n5.3 Experiments . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 48\n5.3.1 Experimental Conditions . . . . . . . . . . . . . . . . . . . . . 49\n5.3.2 Experimental Results . . . . . . . . . . . . . . . . . . . . . . . 50\n5.4 Chapter Summary . . . . . . . . . . . . . . . . . . . . . . . . . . . . 50\n\n6 Conclusions and Future Work 52\n6.1 Conclusions . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 52\n6.2 Future Work . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 52","subitem_description_type":"Other"}]},"item_113_description_25":{"attribute_name":"注記","attribute_value_mlt":[{"subitem_description":"指導教員 : 小林貴訓","subitem_description_type":"Other"}]},"item_113_description_33":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"subitem_description":"text","subitem_description_type":"Other"}]},"item_113_description_34":{"attribute_name":"フォーマット","attribute_value_mlt":[{"subitem_description":"application/pdf","subitem_description_type":"Other"}]},"item_113_dissertation_number_19":{"attribute_name":"学位授与番号","attribute_value_mlt":[{"subitem_dissertationnumber":"甲第1177号"}]},"item_113_identifier_registration":{"attribute_name":"ID登録","attribute_value_mlt":[{"subitem_identifier_reg_text":"10.24561/00019355","subitem_identifier_reg_type":"JaLC"}]},"item_113_publisher_11":{"attribute_name":"出版者名","attribute_value_mlt":[{"subitem_publisher":"埼玉大学大学院理工学研究科"}]},"item_113_publisher_12":{"attribute_name":"出版者名（別言語）","attribute_value_mlt":[{"subitem_publisher":"Graduate School of Science and Engineering, Saitama University"}]},"item_113_record_name_8":{"attribute_name":"書誌","attribute_value_mlt":[{"subitem_record_name":"博士論文（埼玉大学大学院理工学研究科（博士後期課程））"}]},"item_113_text_31":{"attribute_name":"版","attribute_value_mlt":[{"subitem_text_value":"[出版社版]"}]},"item_113_text_36":{"attribute_name":"アイテムID","attribute_value_mlt":[{"subitem_text_value":"GD0001267"}]},"item_113_text_4":{"attribute_name":"著者 所属","attribute_value_mlt":[{"subitem_text_value":"埼玉大学大学院理工学研究科（博士後期課程）理工学専攻"}]},"item_113_text_5":{"attribute_name":"著者 所属（別言語）","attribute_value_mlt":[{"subitem_text_value":"Graduate School of Science and Engineering, Saitama University"}]},"item_113_version_type_32":{"attribute_name":"著者版フラグ","attribute_value_mlt":[{"subitem_version_resource":"http://purl.org/coar/version/c_970fb48d4fbd8a85","subitem_version_type":"VoR"}]},"item_access_right":{"attribute_name":"アクセス権","attribute_value_mlt":[{"subitem_access_right":"open access","subitem_access_right_uri":"http://purl.org/coar/access_right/c_abf2"}]},"item_creator":{"attribute_name":"著者","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"ISLAM, MD MATIQUL","creatorNameLang":"en"},{"creatorName":"イスラム, エムディー マティクル","creatorNameLang":"ja-Kana"}]}]},"item_files":{"attribute_name":"ファイル情報","attribute_type":"file","attribute_value_mlt":[{"accessrole":"open_date","date":[{"dateType":"Available","dateValue":"2021-09-16"}],"displaytype":"detail","filename":"GD0001267.pdf","filesize":[{"value":"3.7 MB"}],"format":"application/pdf","licensetype":"license_note","mimetype":"application/pdf","url":{"label":"GD0001267.pdf","objectType":"fulltext","url":"https://sucra.repo.nii.ac.jp/record/19386/files/GD0001267.pdf"},"version_id":"aa969580-8035-460d-a239-ce5c18a98eb4"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"LiDAR","subitem_subject_scheme":"Other"},{"subitem_subject":"Histogram matching","subitem_subject_scheme":"Other"},{"subitem_subject":"Omnidirectional camera","subitem_subject_scheme":"Other"},{"subitem_subject":"GRU","subitem_subject_scheme":"Other"},{"subitem_subject":"Shopping behavior","subitem_subject_scheme":"Other"},{"subitem_subject":"OpenPose","subitem_subject_scheme":"Other"},{"subitem_subject":"Head orientation","subitem_subject_scheme":"Other"},{"subitem_subject":"Body orientation","subitem_subject_scheme":"Other"},{"subitem_subject":"DNN","subitem_subject_scheme":"Other"},{"subitem_subject":"Kinect camera","subitem_subject_scheme":"Other"},{"subitem_subject":"supermarket","subitem_subject_scheme":"Other"},{"subitem_subject":"person following","subitem_subject_scheme":"Other"},{"subitem_subject":"elderly","subitem_subject_scheme":"Other"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"eng"}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourcetype":"doctoral thesis","resourceuri":"http://purl.org/coar/resource_type/c_db06"}]},"item_title":"An Intelligent Person Following Shopping Support Robot for the Elderly","item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"An Intelligent Person Following Shopping Support Robot for the Elderly","subitem_title_language":"en"}]},"item_type_id":"113","owner":"15","path":["992"],"pubdate":{"attribute_name":"PubDate","attribute_value":"2021-09-16"},"publish_date":"2021-09-16","publish_status":"0","recid":"19386","relation_version_is_last":true,"title":["An Intelligent Person Following Shopping Support Robot for the Elderly"],"weko_creator_id":"15","weko_shared_id":-1},"updated":"2023-06-23T09:01:30.372747+00:00"}