"use strict";(self.webpackChunk_toloka_portal_gatsby_renderer=self.webpackChunk_toloka_portal_gatsby_renderer||[]).push([[9989],{34151:function(e,t,n){n.d(t,{r:function(){return P}});var i=n(4942),r=n(37990),a=n(93252),s=n(52893),o=n(74533),l=n(88586),c=n(88709),d=n(51512),h=n(92401),m=n(44172),u=n(25109),p=n(18987),f=n(13598),g=n(62769),x=n(67294),v=n(3468),j=n(81547),w=n(85893);function y(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function k(e){for(var t=1;tt.date?1:-1})).slice(0,4)}),[P]);return(0,w.jsxs)(w.Fragment,{children:[(0,w.jsx)(s.q,{og:{img:"https://tlk-infra-front.azureedge.net/portal-static/images/page-head/eventsOg/"+t+".jpg"}}),(0,w.jsx)(o.n,{}),(0,w.jsxs)(l.i,{headerPadding:!0,children:[(0,w.jsx)(r.h,{defaultBackground:"dark"}),y,(D||A&&+new Date(A)>q)&&!1===t.startsWith("watch")&&_?(0,w.jsxs)(w.Fragment,{children:[(0,w.jsx)(d.w,{offset:40}),(0,w.jsx)(u.L,{header:{title:O,info:C},theme:"sandSection",id:"registration_form",children:(0,w.jsx)(p.G,{id:_,slug:F||(j.j?window.location.href+"?registered=true":""),leadSource:"Event Registration (website)",textareaPlaceholder:N,textareaRequired:S})})]}):null,(0,w.jsx)(d.w,{offset:40}),X.length>0?(0,w.jsx)(u.L,{header:{title:"Similar "+(0,g.s)(n)+"s"},children:(0,w.jsx)(m.X,{events:X.map((function(e){return e}))})}):null,(0,w.jsx)(h.F,{leftColumn:{title:{text:"Don't miss"},text:(0,w.jsxs)(w.Fragment,{children:["Be sure to attend our informative workshops,"," ",(0,w.jsx)(v.p.Desktop,{children:(0,w.jsx)("br",{})}),"tutorials, and webinars."]})},rightColumn:{input:{label:"Subscribe to updates",placeholder:"Your e-mail"},button:{text:"Get updates",view:"action",type:"submit",size:"l"},successText:"All set! You’ll get an email from us shortly — just click the link to confirm. If it doesn’t show up, check your spam folder.",sendingData:{address:"/api/sender/subscribe",listId:"8AR2Z6Y3-R251"}}}),(0,w.jsx)(d.w,{offset:40}),(0,w.jsx)(a.$,{})]}),(0,w.jsx)(c.f,{})]})}},53370:function(e,t,n){n.d(t,{u:function(){return u}});var i="bD_Z",r="bD_q",a="bD_6",s="bD_h",o="bD_dG",l="bD_dr",c=n(84401),d=n(46237),h=n(57624),m=n(85893),u=function(e){var t=e.title,n=e.button,u=e.text;return(0,m.jsxs)("div",{className:i,children:[(0,m.jsxs)("div",{className:o,children:[(0,m.jsx)(d.D,{level:5,className:r,children:t}),u?(0,m.jsx)("p",{className:s,children:u}):null]}),(0,m.jsx)(c.z,{className:a,view:n.view,type:n.type,size:n.size,url:n.url,onClick:n.onClick,target:n.target,children:n.text}),(0,m.jsx)("div",{className:l,children:(0,m.jsx)(h.F,{})})]})}},94501:function(e,t,n){n.d(t,{B:function(){return S}});var i=n(67294),r="bl_Z",a="bl_c7",s="bl_x",o="bl_w",l=n(79615),c=n(85893),d=function(){return(0,c.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"12",height:"13",fill:"none",children:(0,c.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M1.62.4C.72.4 0 1.13 0 2c0 .87.72 1.58 1.62 1.58A1.6 1.6 0 0 0 3.23 2 1.6 1.6 0 0 0 1.62.4Zm.96 3.62H.19a.2.2 0 0 0-.19.2v7.75c0 .1.09.2.2.2h2.38c.1 0 .19-.1.19-.2V4.22a.2.2 0 0 0-.2-.2Zm4.25.87a2.83 2.83 0 0 1 2.1-.87c2.7 0 3.07 1.92 3.07 3.8v4.16a.2.2 0 0 1-.2.19H9.43a.2.2 0 0 1-.2-.2V8.3c0-1.03-.07-1.82-1.05-1.82-.9 0-1.24.5-1.24 1.76v3.75a.2.2 0 0 1-.2.19H4.36a.2.2 0 0 1-.2-.2V4.4c0-.1.09-.19.2-.19h2.3c.1 0 .18.09.18.2v.48Z",fill:"#fff"})})},h=n(4511),m=n(46237),u="dX_bk",p="dX_bl",f="dX_r",g="dX_bp",x="dX_q",v="dX_bm",j="dX_bn",w="dX_j",y="dX_g",k=function(e){var t=e.imageSrc,n=e.name,i=e.company,r=e.info,a=e.link;return(0,c.jsxs)("div",{className:u,children:[(0,c.jsx)("div",{className:p,children:(0,c.jsx)(l.P,{version:"v1",sizes:[560],imageId:t,className:f})}),(0,c.jsxs)("div",{className:g,children:[(0,c.jsx)(m.D,{className:x,level:5,children:n}),(0,c.jsx)("span",{className:v,children:i}),(0,c.jsx)("span",{className:j,children:r}),(0,c.jsxs)(h.r,{href:a.href,iconPosition:a.iconPosition,target:"_blank",className:w,children:[(0,c.jsx)("div",{className:y,children:(0,c.jsx)(d,{})}),a.text]})]})]})},b="c9_by",P="c9_h",D="c9_g2",_="c9_g3",F="c9_bl",N=function(e){var t=e.text,n=e.imageSrc,i=e.name,r=e.company,a=e.info,s=e.link;return(0,c.jsxs)(c.Fragment,{children:[(0,c.jsx)("div",{className:b}),(0,c.jsxs)("div",{children:[(0,c.jsx)("p",{className:P,children:t}),n&&i&&r&&a&&s?(0,c.jsxs)("button",{className:D,children:[(0,c.jsx)("div",{className:F,children:(0,c.jsx)(l.P,{version:"v1",sizes:[560],imageId:n})}),(0,c.jsx)("span",{children:i}),(0,c.jsx)("div",{className:_,children:(0,c.jsx)(k,{name:i,imageSrc:n,company:r,info:a,link:s})})]}):null]})]})},S=function(e){var t=e.content;return(0,c.jsx)("div",{className:r,children:(0,c.jsx)("div",{className:a,children:t.map((function(e,t){return(0,c.jsxs)(i.Fragment,{children:[(0,c.jsx)("div",{className:s,children:(0,c.jsx)("div",{children:e.date})}),(0,c.jsx)("div",{className:o,children:(0,c.jsx)(N,{text:e.text,imageSrc:e.imageSrc,name:e.name,company:e.company,info:e.info,link:e.link})})]},t)}))})})}},86784:function(e,t,n){n.d(t,{_:function(){return c}});var i=n(84401),r="b6_dx",a="b6_q",s="b6_fq",o=n(85893),l=function(e){var t=e.title,n=e.link;return(0,o.jsxs)("div",{className:r,children:[(0,o.jsx)("div",{className:a,children:t}),(0,o.jsx)("div",{children:(0,o.jsx)(i.z,{size:"m",themeColor:"light",type:"link",url:n,view:"pseudo",children:"Download"})})]})},c=function(e){var t=e.files;return(0,o.jsx)("div",{className:s,children:t.map((function(e){return(0,o.jsx)(l,{title:e.title,link:e.link},e.link)}))})}},18987:function(e,t,n){n.d(t,{G:function(){return g}});var i=n(84181),r=n(79700),a=n.n(r),s=n(84401),o=n(3468),l=n(29005),c=n(34305),d=n(54269),h=n(4511),m=n(67294),u=n(96268),p=n(29499),f=n(85893),g=function(e){var t=e.slug,n=e.id,r=e.isTestForm,g=e.leadSource,x=void 0===g?"WebsiteRegistrationForm":g,v=e.textareaRequired,j=void 0!==v&&v,w=e.textareaPlaceholder,y=void 0===w?"Please drop a few lines about your project":w,k=e.companyRequired,b=void 0===k||k,P=e.jobTitleRequired,D=void 0===P||P,_=e.additionalInputs,F=e.withoutTextArea,N=e.submitButtonText,S=void 0===N?"Submit":N,I=(0,p.useLocation)();(0,m.useEffect)((function(){I.search&&document.querySelectorAll("[id^='utm']").forEach((function(e){e&&(e.value=(0,u.t)(e.id)||"")}))}),[I.search]);var O=(0,m.useMemo)((function(){try{return new URLSearchParams(I.search).get("referer")||void 0}catch(e){return}}),[I.search]);return(0,f.jsxs)("form",{action:"https://webto.salesforce.com/servlet/servlet.WebToLead?encoding=UTF-8",method:"POST",children:[(0,f.jsx)("input",{type:"hidden",name:"oid",defaultValue:"00D7Q000001MbTr"}),(0,f.jsx)("input",{type:"hidden",name:"retURL",defaultValue:t.startsWith("http")?t:"https://toloka.ai/"+t}),(0,f.jsx)("input",{type:"hidden",name:"Campaign_ID",value:n}),(0,f.jsx)("input",{type:"hidden",name:"recordType",id:"recordType",value:"0127Q0000002WpG"}),(0,f.jsx)("input",{id:"00N7Q00000EpH4g",name:"00N7Q00000EpH4g",type:"hidden",value:O}),I.search?(0,f.jsxs)(f.Fragment,{children:[(0,f.jsx)("input",{id:"utm_source",name:"00N7Q000005OX3o",type:"hidden"}),(0,f.jsx)("input",{id:"utm_medium",name:"00N7Q000005OX3j",type:"hidden"}),(0,f.jsx)("input",{id:"utm_term",name:"00N7Q000005OX43",type:"hidden"}),(0,f.jsx)("input",{id:"utm_campaign",name:"00N7Q000005OX3t",type:"hidden"}),(0,f.jsx)("input",{id:"utm_content",name:"00N7Q000005OX3y",type:"hidden"})]}):null,(0,f.jsxs)("div",{className:a()(i.Z.inputsWrap,"light"),children:[(0,f.jsx)(l.I,{placeholder:"First name*",name:"first_name",required:!0}),(0,f.jsx)(l.I,{placeholder:"Last name*",name:"last_name",required:!0}),(0,f.jsx)(l.I,{placeholder:b?"Company*":"Company",name:"company",required:b}),r?null:(0,f.jsx)(l.I,{placeholder:D?"Job title*":"Job title",name:"title",required:D}),(0,f.jsx)(l.I,{placeholder:"Phone number",name:"phone",id:"phone",type:"tel"}),(0,f.jsx)(l.I,{type:"email",placeholder:"Work email*",name:"email",required:!0}),_]}),r||F?null:(0,f.jsx)(d.g,{placeholder:y,id:"00N7Q00000CkEBw",name:"00N7Q00000CkEBw",required:j,className:i.Z.textarea}),(0,f.jsx)(c.X,{label:r?(0,f.jsxs)(f.Fragment,{children:["I consent to allow Intertech Services AG and its affiliates to process my personal information provided in this form under the conditions"," ",(0,f.jsx)(o.p.Desktop,{children:(0,f.jsx)("br",{})}),"and for the purposes defined in the Privacy Policy and Customer Service Agreement, as well as to contact me for follow-up to this request."]}):(0,f.jsxs)(f.Fragment,{children:["I consent to allow Intertech Services AG (Werftestrasse 4, 6005 Luzern, Switzerland) to process my personal information provided in this form to contact me for follow-up to this request. If at any point you wish to withdraw your consent, please fill out the form"," ",(0,f.jsx)(h.r,{target:"_blank",href:"/unsubscribe/",children:"here"}),". For more information about what we do with personal data see our"," ",(0,f.jsx)(h.r,{target:"_blank",href:"/legal/confidential/",children:"privacy notice"}),"."]}),required:!0,className:i.Z.checkbox}),(0,f.jsx)(c.X,{label:r?(0,f.jsxs)(f.Fragment,{children:["I agree to receive advertising and other marketing emails from Intertech Services AG in accordance with the Privacy Policy"," ",(0,f.jsx)(o.p.Desktop,{children:(0,f.jsx)("br",{})})," ","and Customer Service Agreement, including newsletters, invitations to events, and promotional emails."]}):(0,f.jsxs)(f.Fragment,{children:["I agree to receive advertising and other marketing emails from Intertech Services AG (Werftestrasse 4, 6005 Luzern, Switzerland), including newsletters, invitations to events, and promotional emails. If you wish to unsubscribe you can do so by clicking on the unsubscribe link at the end of any email you have received from us to your email address. For more information about what we do with personal data see our"," ",(0,f.jsx)(h.r,{target:"_blank",href:"/legal/confidential/",children:"privacy notice"}),". For more information please click"," ",(0,f.jsx)(h.r,{target:"_blank",href:"/legal/e-newsletter/",children:"here"}),"."]}),id:"00N7Q00000CkDzy",name:"00N7Q00000CkDzy",className:i.Z.checkbox,setValue:"Allow",defaultValue:"Forbid"}),(0,f.jsx)(s.z,{view:"action",type:"submit",className:i.Z.button,children:S}),(0,f.jsx)("input",{id:"lead_source",name:"lead_source",type:"hidden",value:x})]})}},57624:function(e,t,n){n.d(t,{F:function(){return r}});var i=n(85893),r=function(){return(0,i.jsxs)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"375",height:"120",fill:"none",viewBox:"0 0 375 120",children:[(0,i.jsx)("path",{d:"M375 119.5 0 0h375v119.5Z",fill:"#0069FF",fillOpacity:".1"}),(0,i.jsx)("path",{d:"M375 75.5 0 0h375v75.5Z",fill:"#0069FF",fillOpacity:".3"}),(0,i.jsx)("path",{d:"M375 44.353 0 0h375v44.353Z",fill:"#0069FF",fillOpacity:".5"}),(0,i.jsx)("path",{d:"M375 21 0 0h375v21Z",fill:"#0069FF"})]})}},50338:function(e,t,n){n.r(t),n.d(t,{frontmatter:function(){return u},default:function(){return p}});var i=n(51512),r=n(88555),a=n(53370),s=n(34151),o=n(41427),l=n(46237),c=n(4511),d=n(94501),h=n(86784),m=n(85893),u={slug:"recsys-2022",title:"RecSys 2022: Recommender systems with HITL",description:"In our tutorial, we will share more than six years of our crowdsourcing experience and bridge the gap between crowdsourcing and recommender systems communities.",date:"2022-09-18T14:30:00.000-07:00",category:"conference",hosts:[{imageId:"1662042758210936-17989639992055791661",name:"Fedor Zhdanov"},{imageId:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov"},{imageId:"Kcklg8c2ACDiYI_Ue7hw",name:"Natalia Fedorova"},{imageId:"DwL8BD9XV9XBxkJcqw0O",name:"Nikita Pavlichenko"},{imageId:"1663160667260129-4537519149499731950",name:"Maxim Kunakov"}],isRecording:!1};function p(e){var t=e.data;return(0,m.jsx)(s.r,{data:t,eventCategory:u.category,eventSlug:u.slug,eventDate:u.date,eventIsOnDemand:u.isOnDemand,children:(0,m.jsxs)(r.D,{toc:[{url:"#overview",title:"Overview"},{url:"#agenda",title:"Agenda"},{url:"#speakers",title:"Speakers"},{url:"#slides",title:"Slides"}],form:u.isOnDemand||new Date(u.date)>new Date?(0,m.jsx)(a.u,{title:(0,m.jsxs)(m.Fragment,{children:["Don't miss the webinar, ",(0,m.jsx)("br",{}),"register right now"]}),button:{view:"action",text:"Register now",size:"l",type:"link",url:"https://recsys.acm.org/recsys22/registration/"}}):null,button:{view:"action",text:"Register now",size:"l",type:"link",url:"https://recsys.acm.org/recsys22/registration/"},title:u.title,description:u.description,backLink:{text:"Back to Events",iconPosition:"left",href:"/events"},category:u.category,date:u.date,children:[(0,m.jsx)(l.D,{level:2,styleAs:4,id:"overview",children:"Overview"}),(0,m.jsx)("p",{children:"Today, most recommender systems employ Machine Learning to recommend posts, products, and other items, usually produced by the users. Although the impressive progress in Deep Learning and Reinforcement Learning, we observe that recommendations made by such systems still do not correlate with actual human preferences."}),(0,m.jsx)("p",{children:"In our tutorial, we will share more than six years of our crowdsourcing experience and bridge the gap between crowdsourcing and recommender systems communities by showing how one can incorporate human-in-the-loop into their recommender system to gather the real human feedback on the ranked recommendations. We will discuss the ranking data lifecycle and run through it step-by-step. A significant portion of tutorial time is devoted to a hands-on practice, when the attendees will, under our guidance, sample and annotate recommendations on real crowds, build the ground truth dataset, and compute the evaluation scores."}),(0,m.jsx)("p",{children:"All the demonstrated methodology is platform-agnostic and can be freely adapted to a variety of applications. One can gather the judgments on any data labeling platform, from in-house setups till MTurk and Toloka. A related tutorial was previously presented at NAACL-HLT '21, WWW '21, CVPR '20, SIGMOD '20, WSDM '20, and KDD '19."}),(0,m.jsx)("p",{children:"We expect the attendees to understand the core concepts in recommender systems and are able to write short scripts in Python, while we do not require any knowledge of crowdsourcing. We will provide all the necessary definitions and icebreakers to accommodate a wider audience. We recommend the attendees to bring their laptops for the hands-on practice session."}),(0,m.jsx)(i.w,{offset:56}),(0,m.jsx)(l.D,{level:2,styleAs:4,id:"agenda",children:"Agenda"}),(0,m.jsx)(d.B,{content:[{date:"1. Introduction",text:(0,m.jsxs)(m.Fragment,{children:["- Recommender Systems",(0,m.jsx)("br",{}),"- Crowdsourcing",(0,m.jsx)("br",{}),"- Online and Offline Evaluation"]}),imageSrc:"1662042758210936-17989639992055791661",name:"Fedor Zhdanov",info:"Head of ML",company:"Toloka",link:{href:"https://www.linkedin.com/in/fedorzhdanov/",iconPosition:"right",text:"Profile link"}},{date:"2. Ranking and Its Quality",text:(0,m.jsxs)(m.Fragment,{children:["- Problem of Learning-to-Rank",(0,m.jsx)("br",{}),"- Pointwise/Pairwise/Listwise Approaches",(0,m.jsx)("br",{}),"- Evaluation Criteria"]}),imageSrc:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov",info:"Head of Research",company:"Toloka",link:{href:"https://www.linkedin.com/in/ustalov/",iconPosition:"right",text:"Profile link"}},{date:"3. Human-in-the-Loop Essentials",text:(0,m.jsxs)(m.Fragment,{children:["- Core Concepts in Crowdsourcing",(0,m.jsx)("br",{}),"- Quality Control"]}),imageSrc:"DwL8BD9XV9XBxkJcqw0O",name:"Nikita Pavlichenko",info:"Machine Learning Researcher",company:"Toloka",link:{href:"https://www.linkedin.com/in/nikita-pavlichenko/",iconPosition:"right",text:"Profile link"}},{date:"4. Hands-On Practice Session",text:(0,m.jsx)(m.Fragment,{children:"A hands-on practice, when the attendees will, under our guidance, sample and annotate recommendations on real crowds, build the ground truth dataset, and compute the evaluation scores."}),imageSrc:"Kcklg8c2ACDiYI_Ue7hw",name:"Natalia Fedorova",info:"Education Program Manager",company:"Toloka",link:{href:"https://www.linkedin.com/in/natalia-fedorova-8aa008100/",iconPosition:"right",text:"Profile link"}},{date:"5. From Human Labels to Ground Truth",text:(0,m.jsxs)(m.Fragment,{children:["- Problem of Answer Aggregation",(0,m.jsx)("br",{}),"- Pairwise Comparisons",(0,m.jsx)("br",{}),"-"," ",(0,m.jsx)(c.r,{href:"https://github.com/Toloka/crowd-kit",children:"Crowd-Kit Library"})]}),imageSrc:"1662042758210936-17989639992055791661",name:"Fedor Zhdanov",info:"Head of ML",company:"Toloka",link:{href:"https://www.linkedin.com/in/fedorzhdanov/",iconPosition:"right",text:"Profile link"}},{date:"6. Conclusion",text:(0,m.jsxs)(m.Fragment,{children:["- Discussion of Results",(0,m.jsx)("br",{}),"- References"]}),imageSrc:"1662042758210936-17989639992055791661",name:"Fedor Zhdanov",info:"Head of ML",company:"Toloka",link:{href:"https://www.linkedin.com/in/fedorzhdanov/",iconPosition:"right",text:"Profile link"}}]}),(0,m.jsx)(i.w,{offset:80}),(0,m.jsx)(l.D,{level:2,styleAs:4,id:"speakers",children:"Speakers"}),(0,m.jsx)(o.B,{people:[{imgSrc:"1662042758210936-17989639992055791661",name:"Fedor Zhdanov",info:"Head of ML",company:"Toloka",link:{href:"https://www.linkedin.com/in/fedorzhdanov/",iconPosition:"right",text:"Profile link"}},{imgSrc:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov",info:"Head of Research",company:"Toloka",link:{href:"https://www.linkedin.com/in/ustalov/",iconPosition:"right",text:"Profile link"}},{imgSrc:"Kcklg8c2ACDiYI_Ue7hw",name:"Natalia Fedorova",info:"Education Program Manager",company:"Toloka",link:{href:"https://www.linkedin.com/in/natalia-fedorova-8aa008100/",iconPosition:"right",text:"Profile link"}},{imgSrc:"DwL8BD9XV9XBxkJcqw0O",name:"Nikita Pavlichenko",info:"Machine Learning Researcher",company:"Toloka",link:{href:"https://www.linkedin.com/in/nikita-pavlichenko/",iconPosition:"right",text:"Profile link"}},{imgSrc:"1663160667260129-4537519149499731950",name:"Maxim Kunakov",info:"Crowd Solution Architect",company:"Toloka",link:{href:"https://www.linkedin.com/in/kunakov/",iconPosition:"right",text:"Profile link"}}]}),(0,m.jsx)(i.w,{offset:56}),(0,m.jsx)(l.D,{level:2,styleAs:4,id:"slides",children:"Slides"}),(0,m.jsx)(h._,{files:[{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/recsys-2022/recsys2022_part1.pdf",title:(0,m.jsxs)(m.Fragment,{children:[(0,m.jsx)("b",{children:"Part I:"})," Introduction"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/recsys-2022/recsys2022_part2.pdf",title:(0,m.jsxs)(m.Fragment,{children:[(0,m.jsx)("b",{children:"Part II:"})," Ranking and its Quality"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/recsys-2022/recsys2022_part3.pdf",title:(0,m.jsxs)(m.Fragment,{children:[(0,m.jsx)("b",{children:"Part III:"})," Human-in-the-Loop Essentials"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/recsys-2022/recsys2022_part4.pdf",title:(0,m.jsxs)(m.Fragment,{children:[(0,m.jsx)("b",{children:"Part IV:"})," Hands-On Practice Session"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/recsys-2022/recsys2022_part5.pdf",title:(0,m.jsxs)(m.Fragment,{children:[(0,m.jsx)("b",{children:"Part V:"})," From Human Labels to Ground Truth"]})}]})]})})}}}]);