"use strict";(self.webpackChunk_toloka_portal_gatsby_renderer=self.webpackChunk_toloka_portal_gatsby_renderer||[]).push([[8371],{34151:function(e,t,n){n.d(t,{r:function(){return _}});var i=n(4942),r=n(37990),a=n(93252),s=n(52893),o=n(74533),l=n(88586),c=n(88709),d=n(51512),u=n(92401),h=n(44172),m=n(25109),p=n(18987),f=n(13598),x=n(62769),g=n(67294),j=n(3468),v=n(81547),w=n(85893);function b(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function y(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?b(Object(n),!0).forEach((function(t){(0,i.Z)(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):b(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}var k=(0,w.jsxs)("p",{children:["Whether you're looking for data solutions, have an idea for working together,"," ",(0,w.jsx)(j.p.Desktop,{children:(0,w.jsx)("br",{})}),"or need help getting started — we'd love to hear from you."]}),_=function(e){var t=e.eventSlug,n=e.eventCategory,i=e.eventDate,b=e.children,_=e.data,P=e.eventIsOnDemand,D=e.idForm,S=e.returnPathForm,N=e.textareaPlaceholder,I=e.textareaRequired,O=e.formTitle,F=void 0===O?"Registration form":O,q=e.formDescription,z=void 0===q?k:q,A=(0,g.useState)(),C=A[0],X=A[1],E=new Date,T=E.setDate(E.getDate()-1);(0,g.useEffect)((function(){X(i)}),[]);var L=(0,g.useMemo)((function(){return _.allJavascriptFrontmatter.nodes.map((function(e){return y(y({},e.frontmatter),{},{slug:"/events/"+e.frontmatter.slug})})).concat(f.D).filter((function(e){if(e.category===n&&e.slug!=="/events/"+t&&!0!==e.onlyRecordingsSection)return e.date&&T<+new Date(e.date)||e.isOnDemand})).sort((function(e,t){return e.date&&t.date&&e.date>t.date?1:-1})).slice(0,4)}),[_]);return(0,w.jsxs)(w.Fragment,{children:[(0,w.jsx)(s.q,{og:{img:"https://tlk-infra-front.azureedge.net/portal-static/images/page-head/eventsOg/"+t+".jpg"}}),(0,w.jsx)(o.n,{}),(0,w.jsxs)(l.i,{headerPadding:!0,children:[(0,w.jsx)(r.h,{defaultBackground:"dark"}),b,(P||C&&+new Date(C)>T)&&!1===t.startsWith("watch")&&D?(0,w.jsxs)(w.Fragment,{children:[(0,w.jsx)(d.w,{offset:40}),(0,w.jsx)(m.L,{header:{title:F,info:z},theme:"sandSection",id:"registration_form",children:(0,w.jsx)(p.G,{id:D,slug:S||(v.j?window.location.href+"?registered=true":""),leadSource:"Event Registration (website)",textareaPlaceholder:N,textareaRequired:I})})]}):null,(0,w.jsx)(d.w,{offset:40}),L.length>0?(0,w.jsx)(m.L,{header:{title:"Similar "+(0,x.s)(n)+"s"},children:(0,w.jsx)(h.X,{events:L.map((function(e){return e}))})}):null,(0,w.jsx)(u.F,{leftColumn:{title:{text:"Don't miss"},text:(0,w.jsxs)(w.Fragment,{children:["Be sure to attend our informative workshops,"," ",(0,w.jsx)(j.p.Desktop,{children:(0,w.jsx)("br",{})}),"tutorials, and webinars."]})},rightColumn:{input:{label:"Subscribe to updates",placeholder:"Your e-mail"},button:{text:"Get updates",view:"action",type:"submit",size:"l"},successText:"All set! You’ll get an email from us shortly — just click the link to confirm. If it doesn’t show up, check your spam folder.",sendingData:{address:"/api/sender/subscribe",listId:"8AR2Z6Y3-R251"}}}),(0,w.jsx)(d.w,{offset:40}),(0,w.jsx)(a.$,{})]}),(0,w.jsx)(c.f,{})]})}},94501:function(e,t,n){n.d(t,{B:function(){return I}});var i=n(67294),r="bl_Z",a="bl_c7",s="bl_x",o="bl_w",l=n(79615),c=n(85893),d=function(){return(0,c.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"12",height:"13",fill:"none",children:(0,c.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M1.62.4C.72.4 0 1.13 0 2c0 .87.72 1.58 1.62 1.58A1.6 1.6 0 0 0 3.23 2 1.6 1.6 0 0 0 1.62.4Zm.96 3.62H.19a.2.2 0 0 0-.19.2v7.75c0 .1.09.2.2.2h2.38c.1 0 .19-.1.19-.2V4.22a.2.2 0 0 0-.2-.2Zm4.25.87a2.83 2.83 0 0 1 2.1-.87c2.7 0 3.07 1.92 3.07 3.8v4.16a.2.2 0 0 1-.2.19H9.43a.2.2 0 0 1-.2-.2V8.3c0-1.03-.07-1.82-1.05-1.82-.9 0-1.24.5-1.24 1.76v3.75a.2.2 0 0 1-.2.19H4.36a.2.2 0 0 1-.2-.2V4.4c0-.1.09-.19.2-.19h2.3c.1 0 .18.09.18.2v.48Z",fill:"#fff"})})},u=n(4511),h=n(46237),m="dX_bk",p="dX_bl",f="dX_r",x="dX_bp",g="dX_q",j="dX_bm",v="dX_bn",w="dX_j",b="dX_g",y=function(e){var t=e.imageSrc,n=e.name,i=e.company,r=e.info,a=e.link;return(0,c.jsxs)("div",{className:m,children:[(0,c.jsx)("div",{className:p,children:(0,c.jsx)(l.P,{version:"v1",sizes:[560],imageId:t,className:f})}),(0,c.jsxs)("div",{className:x,children:[(0,c.jsx)(h.D,{className:g,level:5,children:n}),(0,c.jsx)("span",{className:j,children:i}),(0,c.jsx)("span",{className:v,children:r}),(0,c.jsxs)(u.r,{href:a.href,iconPosition:a.iconPosition,target:"_blank",className:w,children:[(0,c.jsx)("div",{className:b,children:(0,c.jsx)(d,{})}),a.text]})]})]})},k="c9_by",_="c9_h",P="c9_g2",D="c9_g3",S="c9_bl",N=function(e){var t=e.text,n=e.imageSrc,i=e.name,r=e.company,a=e.info,s=e.link;return(0,c.jsxs)(c.Fragment,{children:[(0,c.jsx)("div",{className:k}),(0,c.jsxs)("div",{children:[(0,c.jsx)("p",{className:_,children:t}),n&&i&&r&&a&&s?(0,c.jsxs)("button",{className:P,children:[(0,c.jsx)("div",{className:S,children:(0,c.jsx)(l.P,{version:"v1",sizes:[560],imageId:n})}),(0,c.jsx)("span",{children:i}),(0,c.jsx)("div",{className:D,children:(0,c.jsx)(y,{name:i,imageSrc:n,company:r,info:a,link:s})})]}):null]})]})},I=function(e){var t=e.content;return(0,c.jsx)("div",{className:r,children:(0,c.jsx)("div",{className:a,children:t.map((function(e,t){return(0,c.jsxs)(i.Fragment,{children:[(0,c.jsx)("div",{className:s,children:(0,c.jsx)("div",{children:e.date})}),(0,c.jsx)("div",{className:o,children:(0,c.jsx)(N,{text:e.text,imageSrc:e.imageSrc,name:e.name,company:e.company,info:e.info,link:e.link})})]},t)}))})})}},86784:function(e,t,n){n.d(t,{_:function(){return c}});var i=n(84401),r="b6_dx",a="b6_q",s="b6_fq",o=n(85893),l=function(e){var t=e.title,n=e.link;return(0,o.jsxs)("div",{className:r,children:[(0,o.jsx)("div",{className:a,children:t}),(0,o.jsx)("div",{children:(0,o.jsx)(i.z,{size:"m",themeColor:"light",type:"link",url:n,view:"pseudo",children:"Download"})})]})},c=function(e){var t=e.files;return(0,o.jsx)("div",{className:s,children:t.map((function(e){return(0,o.jsx)(l,{title:e.title,link:e.link},e.link)}))})}},18987:function(e,t,n){n.d(t,{G:function(){return x}});var i=n(84181),r=n(79700),a=n.n(r),s=n(84401),o=n(3468),l=n(29005),c=n(34305),d=n(54269),u=n(4511),h=n(67294),m=n(96268),p=n(29499),f=n(85893),x=function(e){var t=e.slug,n=e.id,r=e.isTestForm,x=e.leadSource,g=void 0===x?"WebsiteRegistrationForm":x,j=e.textareaRequired,v=void 0!==j&&j,w=e.textareaPlaceholder,b=void 0===w?"Please drop a few lines about your project":w,y=e.companyRequired,k=void 0===y||y,_=e.jobTitleRequired,P=void 0===_||_,D=e.additionalInputs,S=e.withoutTextArea,N=e.submitButtonText,I=void 0===N?"Submit":N,O=(0,p.useLocation)();(0,h.useEffect)((function(){O.search&&document.querySelectorAll("[id^='utm']").forEach((function(e){e&&(e.value=(0,m.t)(e.id)||"")}))}),[O.search]);var F=(0,h.useMemo)((function(){try{return new URLSearchParams(O.search).get("referer")||void 0}catch(e){return}}),[O.search]);return(0,f.jsxs)("form",{action:"https://webto.salesforce.com/servlet/servlet.WebToLead?encoding=UTF-8",method:"POST",children:[(0,f.jsx)("input",{type:"hidden",name:"oid",defaultValue:"00D7Q000001MbTr"}),(0,f.jsx)("input",{type:"hidden",name:"retURL",defaultValue:t.startsWith("http")?t:"https://toloka.ai/"+t}),(0,f.jsx)("input",{type:"hidden",name:"Campaign_ID",value:n}),(0,f.jsx)("input",{type:"hidden",name:"recordType",id:"recordType",value:"0127Q0000002WpG"}),(0,f.jsx)("input",{id:"00N7Q00000EpH4g",name:"00N7Q00000EpH4g",type:"hidden",value:F}),O.search?(0,f.jsxs)(f.Fragment,{children:[(0,f.jsx)("input",{id:"utm_source",name:"00N7Q000005OX3o",type:"hidden"}),(0,f.jsx)("input",{id:"utm_medium",name:"00N7Q000005OX3j",type:"hidden"}),(0,f.jsx)("input",{id:"utm_term",name:"00N7Q000005OX43",type:"hidden"}),(0,f.jsx)("input",{id:"utm_campaign",name:"00N7Q000005OX3t",type:"hidden"}),(0,f.jsx)("input",{id:"utm_content",name:"00N7Q000005OX3y",type:"hidden"})]}):null,(0,f.jsxs)("div",{className:a()(i.Z.inputsWrap,"light"),children:[(0,f.jsx)(l.I,{placeholder:"First name*",name:"first_name",required:!0}),(0,f.jsx)(l.I,{placeholder:"Last name*",name:"last_name",required:!0}),(0,f.jsx)(l.I,{placeholder:k?"Company*":"Company",name:"company",required:k}),r?null:(0,f.jsx)(l.I,{placeholder:P?"Job title*":"Job title",name:"title",required:P}),(0,f.jsx)(l.I,{placeholder:"Phone number",name:"phone",id:"phone",type:"tel"}),(0,f.jsx)(l.I,{type:"email",placeholder:"Work email*",name:"email",required:!0}),D]}),r||S?null:(0,f.jsx)(d.g,{placeholder:b,id:"00N7Q00000CkEBw",name:"00N7Q00000CkEBw",required:v,className:i.Z.textarea}),(0,f.jsx)(c.X,{label:r?(0,f.jsxs)(f.Fragment,{children:["I consent to allow Intertech Services AG and its affiliates to process my personal information provided in this form under the conditions"," ",(0,f.jsx)(o.p.Desktop,{children:(0,f.jsx)("br",{})}),"and for the purposes defined in the Privacy Policy and Customer Service Agreement, as well as to contact me for follow-up to this request."]}):(0,f.jsxs)(f.Fragment,{children:["I consent to allow Intertech Services AG (Werftestrasse 4, 6005 Luzern, Switzerland) to process my personal information provided in this form to contact me for follow-up to this request. If at any point you wish to withdraw your consent, please fill out the form"," ",(0,f.jsx)(u.r,{target:"_blank",href:"/unsubscribe/",children:"here"}),". For more information about what we do with personal data see our"," ",(0,f.jsx)(u.r,{target:"_blank",href:"/legal/confidential/",children:"privacy notice"}),"."]}),required:!0,className:i.Z.checkbox}),(0,f.jsx)(c.X,{label:r?(0,f.jsxs)(f.Fragment,{children:["I agree to receive advertising and other marketing emails from Intertech Services AG in accordance with the Privacy Policy"," ",(0,f.jsx)(o.p.Desktop,{children:(0,f.jsx)("br",{})})," ","and Customer Service Agreement, including newsletters, invitations to events, and promotional emails."]}):(0,f.jsxs)(f.Fragment,{children:["I agree to receive advertising and other marketing emails from Intertech Services AG (Werftestrasse 4, 6005 Luzern, Switzerland), including newsletters, invitations to events, and promotional emails. If you wish to unsubscribe you can do so by clicking on the unsubscribe link at the end of any email you have received from us to your email address. For more information about what we do with personal data see our"," ",(0,f.jsx)(u.r,{target:"_blank",href:"/legal/confidential/",children:"privacy notice"}),". For more information please click"," ",(0,f.jsx)(u.r,{target:"_blank",href:"/legal/e-newsletter/",children:"here"}),"."]}),id:"00N7Q00000CkDzy",name:"00N7Q00000CkDzy",className:i.Z.checkbox,setValue:"Allow",defaultValue:"Forbid"}),(0,f.jsx)(s.z,{view:"action",type:"submit",className:i.Z.button,children:I}),(0,f.jsx)("input",{id:"lead_source",name:"lead_source",type:"hidden",value:g})]})}},41836:function(e,t,n){n.r(t),n.d(t,{frontmatter:function(){return u},default:function(){return h}});var i=n(51512),r=n(88555),a=n(34151),s=n(41427),o=n(46237),l=n(94501),c=n(86784),d=n(85893),u={slug:"tutorial-www",title:"WWW 2021: Web ranking with Human-in-the-Loop",description:"In our tutorial, we present a systematic view on using Human-in-the-Loop to obtain scalable offline evaluation processes and, in particular, high-quality relevance judgements.",date:"2021-04-13T10:00:00.000Z",category:"conference",hosts:[{imageId:"CV2fFNwtYAmTaFnMcVTc",name:"Alexey Drutsa"},{imageId:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov"},{imageId:"YJ7-107LJUdfxbpwJB5E",name:"Daria Baidakova"},{imageId:"cP8sg88uPV0Pi2Q8nXAX",name:"Nikita Popov"}]};function h(e){var t=e.data;return(0,d.jsx)(a.r,{data:t,eventCategory:u.category,eventSlug:u.slug,eventDate:u.date,eventIsOnDemand:u.isOnDemand,children:(0,d.jsxs)(r.D,{toc:[{url:"#overview",title:"Overview"},{url:"#speakers",title:"Speakers"},{url:"#schedule",title:"Schedule"},{url:"#slides",title:"Slides"}],title:u.title,description:u.description,backLink:{text:"Back to Events",iconPosition:"left",href:"/events"},category:u.category,date:u.date,children:[(0,d.jsx)(o.D,{level:2,styleAs:4,id:"overview",children:"Overview"}),(0,d.jsx)("p",{children:"Modern Web services widely employ sophisticated Machine Learning techniques to rank news, posts, products, and other items presented to the users or contributed by them. These techniques are usually built on offline data pipelines and use a numerical approximation of the relevance of the demonstrated content. In our hands-on tutorial, we present a systematic view on using Human-in-the-Loop to obtain scalable offline evaluation processes and, in particular, high-quality relevance judgements. We will introduce the ranking problem to the attendees, discuss the commonly used ranking quality metrics, and then focus on Human-in-the-Loop-based approach to obtain relevance judgements at scale. More precisely, we will present a thorough introduction to pairwise comparisons, demonstrate how these comparisons can be obtained using Crowdsourcing, and organize a hands-on practice session in which the attendees will obtain high-quality relevance judgements for search quality evaluation. Finally, we will discuss the obtained relevance judgements, point out directions for further studies, and answer questions asked during the tutorial."}),(0,d.jsx)(i.w,{offset:56}),(0,d.jsx)(o.D,{level:2,styleAs:4,id:"speakers",children:"Speakers"}),(0,d.jsx)(s.B,{people:[{imgSrc:"CV2fFNwtYAmTaFnMcVTc",name:"Alexey Drutsa",company:"Toloka",info:"Head of Efficiency & Growth Division"},{imgSrc:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov",company:"Toloka",info:"Analyst / Software Developer"},{imgSrc:"YJ7-107LJUdfxbpwJB5E",name:"Daria Baidakova",company:"Toloka",info:"Director of Educational Programs"},{imgSrc:"cP8sg88uPV0Pi2Q8nXAX",name:"Nikita Popov",company:"Yandex",info:"Senior Data Scientist"}]}),(0,d.jsx)(i.w,{offset:80}),(0,d.jsx)(o.D,{level:2,styleAs:4,id:"schedule",children:"Schedule"}),(0,d.jsx)(l.B,{content:[{date:"09:00 - 09:45",text:(0,d.jsx)("b",{children:"Introduction to Offline Evaluation"})},{date:"09:45 - 10:45",text:(0,d.jsx)("b",{children:"Ranking and Quality Metrics"})},{date:"10:45 - 11:30",text:(0,d.jsx)("b",{children:"Human-in-the-Loop Essentials"})},{date:"11:30 - 13:00",text:(0,d.jsx)("b",{children:"Hands-On Practice Session I"})},{date:"13:00 - 14:00",text:(0,d.jsx)("b",{children:"Lunch Break"})},{date:"14:00 - 14:30",text:(0,d.jsx)("b",{children:"Hands-On Practice Session II"})},{date:"14:30 - 15:30",text:(0,d.jsx)("b",{children:"Pairwise Comparisons"})},{date:"15:30 - 16:00",text:(0,d.jsx)("b",{children:"Final Remarks and Conclusion"})}]}),(0,d.jsx)(i.w,{offset:80}),(0,d.jsx)(o.D,{level:2,styleAs:4,id:"slides",children:"Slides"}),(0,d.jsx)(c._,{files:[{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/www-2021/www2021_part0.pdf",title:(0,d.jsxs)(d.Fragment,{children:[(0,d.jsx)("b",{children:"Part 0:"})," Introduction to Offline Evaluation"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/www-2021/www2021_part1.pdf",title:(0,d.jsxs)(d.Fragment,{children:[(0,d.jsx)("b",{children:"Part I:"})," Ranking and Quality Metrics"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/www-2021/www2021_part2.pdf",title:(0,d.jsxs)(d.Fragment,{children:[(0,d.jsx)("b",{children:"Part II:"})," Human-in-the-Loop Essentials"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/www-2021/www2021_part3.pdf",title:(0,d.jsxs)(d.Fragment,{children:[(0,d.jsx)("b",{children:"Part III - IV:"})," Hands-On Practice Session"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/www-2021/www2021_part5.pdf",title:(0,d.jsxs)(d.Fragment,{children:[(0,d.jsx)("b",{children:"Part V:"})," Pairwise Comparisons"]})}]})]})})}}}]);