"use strict";(self.webpackChunk_toloka_portal_gatsby_renderer=self.webpackChunk_toloka_portal_gatsby_renderer||[]).push([[1798],{34151:function(e,t,r){r.d(t,{r:function(){return _}});var n=r(4942),a=r(37990),i=r(93252),s=r(52893),o=r(74533),l=r(88586),c=r(88709),d=r(51512),u=r(92401),m=r(44172),f=r(25109),p=r(18987),h=r(13598),g=r(62769),x=r(67294),j=r(3468),v=r(81547),b=r(85893);function k(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function y(e){for(var t=1;tt.date?1:-1})).slice(0,4)}),[_]);return(0,b.jsxs)(b.Fragment,{children:[(0,b.jsx)(s.q,{og:{img:"https://tlk-infra-front.azureedge.net/portal-static/images/page-head/eventsOg/"+t+".jpg"}}),(0,b.jsx)(o.n,{}),(0,b.jsxs)(l.i,{headerPadding:!0,children:[(0,b.jsx)(a.h,{defaultBackground:"dark"}),k,(D||C&&+new Date(C)>E)&&!1===t.startsWith("watch")&&P?(0,b.jsxs)(b.Fragment,{children:[(0,b.jsx)(d.w,{offset:40}),(0,b.jsx)(f.L,{header:{title:T,info:z},theme:"sandSection",id:"registration_form",children:(0,b.jsx)(p.G,{id:P,slug:F||(v.j?window.location.href+"?registered=true":""),leadSource:"Event Registration (website)",textareaPlaceholder:I,textareaRequired:N})})]}):null,(0,b.jsx)(d.w,{offset:40}),R.length>0?(0,b.jsx)(f.L,{header:{title:"Similar "+(0,g.s)(r)+"s"},children:(0,b.jsx)(m.X,{events:R.map((function(e){return e}))})}):null,(0,b.jsx)(u.F,{leftColumn:{title:{text:"Don't miss"},text:(0,b.jsxs)(b.Fragment,{children:["Be sure to attend our informative workshops,"," ",(0,b.jsx)(j.p.Desktop,{children:(0,b.jsx)("br",{})}),"tutorials, and webinars."]})},rightColumn:{input:{label:"Subscribe to updates",placeholder:"Your e-mail"},button:{text:"Get updates",view:"action",type:"submit",size:"l"},successText:"All set! You’ll get an email from us shortly — just click the link to confirm. If it doesn’t show up, check your spam folder.",sendingData:{address:"/api/sender/subscribe",listId:"8AR2Z6Y3-R251"}}}),(0,b.jsx)(d.w,{offset:40}),(0,b.jsx)(i.$,{})]}),(0,b.jsx)(c.f,{})]})}},94501:function(e,t,r){r.d(t,{B:function(){return N}});var n=r(67294),a="bl_Z",i="bl_c7",s="bl_x",o="bl_w",l=r(79615),c=r(85893),d=function(){return(0,c.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"12",height:"13",fill:"none",children:(0,c.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M1.62.4C.72.4 0 1.13 0 2c0 .87.72 1.58 1.62 1.58A1.6 1.6 0 0 0 3.23 2 1.6 1.6 0 0 0 1.62.4Zm.96 3.62H.19a.2.2 0 0 0-.19.2v7.75c0 .1.09.2.2.2h2.38c.1 0 .19-.1.19-.2V4.22a.2.2 0 0 0-.2-.2Zm4.25.87a2.83 2.83 0 0 1 2.1-.87c2.7 0 3.07 1.92 3.07 3.8v4.16a.2.2 0 0 1-.2.19H9.43a.2.2 0 0 1-.2-.2V8.3c0-1.03-.07-1.82-1.05-1.82-.9 0-1.24.5-1.24 1.76v3.75a.2.2 0 0 1-.2.19H4.36a.2.2 0 0 1-.2-.2V4.4c0-.1.09-.19.2-.19h2.3c.1 0 .18.09.18.2v.48Z",fill:"#fff"})})},u=r(4511),m=r(46237),f="dX_bk",p="dX_bl",h="dX_r",g="dX_bp",x="dX_q",j="dX_bm",v="dX_bn",b="dX_j",k="dX_g",y=function(e){var t=e.imageSrc,r=e.name,n=e.company,a=e.info,i=e.link;return(0,c.jsxs)("div",{className:f,children:[(0,c.jsx)("div",{className:p,children:(0,c.jsx)(l.P,{version:"v1",sizes:[560],imageId:t,className:h})}),(0,c.jsxs)("div",{className:g,children:[(0,c.jsx)(m.D,{className:x,level:5,children:r}),(0,c.jsx)("span",{className:j,children:n}),(0,c.jsx)("span",{className:v,children:a}),(0,c.jsxs)(u.r,{href:i.href,iconPosition:i.iconPosition,target:"_blank",className:b,children:[(0,c.jsx)("div",{className:k,children:(0,c.jsx)(d,{})}),i.text]})]})]})},w="c9_by",_="c9_h",D="c9_g2",P="c9_g3",F="c9_bl",I=function(e){var t=e.text,r=e.imageSrc,n=e.name,a=e.company,i=e.info,s=e.link;return(0,c.jsxs)(c.Fragment,{children:[(0,c.jsx)("div",{className:w}),(0,c.jsxs)("div",{children:[(0,c.jsx)("p",{className:_,children:t}),r&&n&&a&&i&&s?(0,c.jsxs)("button",{className:D,children:[(0,c.jsx)("div",{className:F,children:(0,c.jsx)(l.P,{version:"v1",sizes:[560],imageId:r})}),(0,c.jsx)("span",{children:n}),(0,c.jsx)("div",{className:P,children:(0,c.jsx)(y,{name:n,imageSrc:r,company:a,info:i,link:s})})]}):null]})]})},N=function(e){var t=e.content;return(0,c.jsx)("div",{className:a,children:(0,c.jsx)("div",{className:i,children:t.map((function(e,t){return(0,c.jsxs)(n.Fragment,{children:[(0,c.jsx)("div",{className:s,children:(0,c.jsx)("div",{children:e.date})}),(0,c.jsx)("div",{className:o,children:(0,c.jsx)(I,{text:e.text,imageSrc:e.imageSrc,name:e.name,company:e.company,info:e.info,link:e.link})})]},t)}))})})}},86784:function(e,t,r){r.d(t,{_:function(){return c}});var n=r(84401),a="b6_dx",i="b6_q",s="b6_fq",o=r(85893),l=function(e){var t=e.title,r=e.link;return(0,o.jsxs)("div",{className:a,children:[(0,o.jsx)("div",{className:i,children:t}),(0,o.jsx)("div",{children:(0,o.jsx)(n.z,{size:"m",themeColor:"light",type:"link",url:r,view:"pseudo",children:"Download"})})]})},c=function(e){var t=e.files;return(0,o.jsx)("div",{className:s,children:t.map((function(e){return(0,o.jsx)(l,{title:e.title,link:e.link},e.link)}))})}},18987:function(e,t,r){r.d(t,{G:function(){return g}});var n=r(84181),a=r(79700),i=r.n(a),s=r(84401),o=r(3468),l=r(29005),c=r(34305),d=r(54269),u=r(4511),m=r(67294),f=r(96268),p=r(29499),h=r(85893),g=function(e){var t=e.slug,r=e.id,a=e.isTestForm,g=e.leadSource,x=void 0===g?"WebsiteRegistrationForm":g,j=e.textareaRequired,v=void 0!==j&&j,b=e.textareaPlaceholder,k=void 0===b?"Please drop a few lines about your project":b,y=e.companyRequired,w=void 0===y||y,_=e.jobTitleRequired,D=void 0===_||_,P=e.additionalInputs,F=e.withoutTextArea,I=e.submitButtonText,N=void 0===I?"Submit":I,S=(0,p.useLocation)();(0,m.useEffect)((function(){S.search&&document.querySelectorAll("[id^='utm']").forEach((function(e){e&&(e.value=(0,f.t)(e.id)||"")}))}),[S.search]);var T=(0,m.useMemo)((function(){try{return new URLSearchParams(S.search).get("referer")||void 0}catch(e){return}}),[S.search]);return(0,h.jsxs)("form",{action:"https://webto.salesforce.com/servlet/servlet.WebToLead?encoding=UTF-8",method:"POST",children:[(0,h.jsx)("input",{type:"hidden",name:"oid",defaultValue:"00D7Q000001MbTr"}),(0,h.jsx)("input",{type:"hidden",name:"retURL",defaultValue:t.startsWith("http")?t:"https://toloka.ai/"+t}),(0,h.jsx)("input",{type:"hidden",name:"Campaign_ID",value:r}),(0,h.jsx)("input",{type:"hidden",name:"recordType",id:"recordType",value:"0127Q0000002WpG"}),(0,h.jsx)("input",{id:"00N7Q00000EpH4g",name:"00N7Q00000EpH4g",type:"hidden",value:T}),S.search?(0,h.jsxs)(h.Fragment,{children:[(0,h.jsx)("input",{id:"utm_source",name:"00N7Q000005OX3o",type:"hidden"}),(0,h.jsx)("input",{id:"utm_medium",name:"00N7Q000005OX3j",type:"hidden"}),(0,h.jsx)("input",{id:"utm_term",name:"00N7Q000005OX43",type:"hidden"}),(0,h.jsx)("input",{id:"utm_campaign",name:"00N7Q000005OX3t",type:"hidden"}),(0,h.jsx)("input",{id:"utm_content",name:"00N7Q000005OX3y",type:"hidden"})]}):null,(0,h.jsxs)("div",{className:i()(n.Z.inputsWrap,"light"),children:[(0,h.jsx)(l.I,{placeholder:"First name*",name:"first_name",required:!0}),(0,h.jsx)(l.I,{placeholder:"Last name*",name:"last_name",required:!0}),(0,h.jsx)(l.I,{placeholder:w?"Company*":"Company",name:"company",required:w}),a?null:(0,h.jsx)(l.I,{placeholder:D?"Job title*":"Job title",name:"title",required:D}),(0,h.jsx)(l.I,{placeholder:"Phone number",name:"phone",id:"phone",type:"tel"}),(0,h.jsx)(l.I,{type:"email",placeholder:"Work email*",name:"email",required:!0}),P]}),a||F?null:(0,h.jsx)(d.g,{placeholder:k,id:"00N7Q00000CkEBw",name:"00N7Q00000CkEBw",required:v,className:n.Z.textarea}),(0,h.jsx)(c.X,{label:a?(0,h.jsxs)(h.Fragment,{children:["I consent to allow Intertech Services AG and its affiliates to process my personal information provided in this form under the conditions"," ",(0,h.jsx)(o.p.Desktop,{children:(0,h.jsx)("br",{})}),"and for the purposes defined in the Privacy Policy and Customer Service Agreement, as well as to contact me for follow-up to this request."]}):(0,h.jsxs)(h.Fragment,{children:["I consent to allow Intertech Services AG (Werftestrasse 4, 6005 Luzern, Switzerland) to process my personal information provided in this form to contact me for follow-up to this request. If at any point you wish to withdraw your consent, please fill out the form"," ",(0,h.jsx)(u.r,{target:"_blank",href:"/unsubscribe/",children:"here"}),". For more information about what we do with personal data see our"," ",(0,h.jsx)(u.r,{target:"_blank",href:"/legal/confidential/",children:"privacy notice"}),"."]}),required:!0,className:n.Z.checkbox}),(0,h.jsx)(c.X,{label:a?(0,h.jsxs)(h.Fragment,{children:["I agree to receive advertising and other marketing emails from Intertech Services AG in accordance with the Privacy Policy"," ",(0,h.jsx)(o.p.Desktop,{children:(0,h.jsx)("br",{})})," ","and Customer Service Agreement, including newsletters, invitations to events, and promotional emails."]}):(0,h.jsxs)(h.Fragment,{children:["I agree to receive advertising and other marketing emails from Intertech Services AG (Werftestrasse 4, 6005 Luzern, Switzerland), including newsletters, invitations to events, and promotional emails. If you wish to unsubscribe you can do so by clicking on the unsubscribe link at the end of any email you have received from us to your email address. For more information about what we do with personal data see our"," ",(0,h.jsx)(u.r,{target:"_blank",href:"/legal/confidential/",children:"privacy notice"}),". For more information please click"," ",(0,h.jsx)(u.r,{target:"_blank",href:"/legal/e-newsletter/",children:"here"}),"."]}),id:"00N7Q00000CkDzy",name:"00N7Q00000CkDzy",className:n.Z.checkbox,setValue:"Allow",defaultValue:"Forbid"}),(0,h.jsx)(s.z,{view:"action",type:"submit",className:n.Z.button,children:N}),(0,h.jsx)("input",{id:"lead_source",name:"lead_source",type:"hidden",value:x})]})}},23389:function(e,t,r){r.r(t),r.d(t,{frontmatter:function(){return m},default:function(){return f}});var n=r(51512),a=r(86784),i=r(88555),s=r(34151),o=r(41427),l=r(46237),c=r(94501),d=r(4511),u=r(85893),m={slug:"tutorial-sigmod",title:"Tutorial at SIGMOD 2020",description:"In this tutorial, we present some key techniques for efficiently collecting labeled data, including aggregation, incremental relabeling, and dynamic pricing.",date:"2020-06-14T10:00:00.000Z",category:"conference",hosts:[{imageId:"CV2fFNwtYAmTaFnMcVTc",name:"Alexey Drutsa"},{imageId:"zGHkk2wbT4yuQVbfFaJO",name:"Valentina Fedorova"},{imageId:"VFeyhjwZLWoBnWODaDDa",name:"Olga Megorskaya"},{imageId:"oL04gaQuspW0NPZLsF8N",name:"Evfrosiniya Zerminova"},{imageId:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov"},{imageId:"YJ7-107LJUdfxbpwJB5E",name:"Daria Baidakova"}]};function f(e){var t=e.data;return(0,u.jsx)(s.r,{data:t,eventCategory:m.category,eventSlug:m.slug,eventDate:m.date,eventIsOnDemand:m.isOnDemand,children:(0,u.jsxs)(i.D,{toc:[{url:"#overview",title:"Overview"},{url:"#speakers",title:"Speakers"},{url:"#schedule",title:"Schedule"},{url:"#slides",title:"Slides"}],title:m.title,description:m.description,backLink:{text:"Back to Events",iconPosition:"left",href:"/events"},category:m.category,date:m.date,children:[(0,u.jsx)(l.D,{level:2,styleAs:4,id:"overview",children:"Overview"}),(0,u.jsx)("p",{children:"In this tutorial, we introduce data labeling via public crowdsourcing marketplaces and present some key techniques for efficiently collecting labeled data, including aggregation, incremental relabeling, and dynamic pricing."}),(0,u.jsx)("p",{children:"This is followed by a practice session, where participants choose one real label collection task, experiment with selecting settings for the labeling process, and launch their own label collection project on one of the largest crowdsourcing marketplaces. During the tutorial, all projects are run on the real Toloka crowd. While we are waiting for the crowd performers to annotate participants’ projects, we present the major theoretical results in efficient aggregation, incremental relabeling, and dynamic pricing. We also discuss the strengths and weaknesses of crowdsourcing, as well as applicability to real-world tasks, summarizing our five years of research and industrial expertise in crowdsourcing. All participants receive feedback on their projects and practical advice."}),(0,u.jsx)(n.w,{offset:56}),(0,u.jsx)(l.D,{level:2,styleAs:4,id:"speakers",children:"Speakers"}),(0,u.jsx)(o.B,{people:[{imgSrc:"CV2fFNwtYAmTaFnMcVTc",name:"Alexey Drutsa",company:"Toloka",info:"Head of Efficiency & Growth Division"},{imgSrc:"zGHkk2wbT4yuQVbfFaJO",name:"Valentina Fedorova",company:"Toloka",info:"Analyst"},{imgSrc:"VFeyhjwZLWoBnWODaDDa",name:"Olga Megorskaya",company:"Toloka",info:"CEO"},{imgSrc:"oL04gaQuspW0NPZLsF8N",name:"Evfrosiniya Zerminova",company:"Toloka",info:"Technical Product Manager"},{imgSrc:"Rh4qN8gXmzlgyEkuGvz6",name:"Dmitry Ustalov",company:"Toloka",info:"Head of Research"},{imgSrc:"YJ7-107LJUdfxbpwJB5E",name:"Daria Baidakova",company:"Toloka",info:"Director of Educational Programs"}]}),(0,u.jsx)(n.w,{offset:80}),(0,u.jsx)(l.D,{level:2,styleAs:4,id:"schedule",children:"Schedule"}),(0,u.jsx)(c.B,{content:[{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_introduction.pdf",target:"_blank",children:"Part 0: Introduction"}),text:(0,u.jsxs)(u.Fragment,{children:["— The concept of crowdsourcing ",(0,u.jsx)("br",{}),"— Crowdsourcing task examples",(0,u.jsx)("br",{}),"— Crowdsourcing platforms",(0,u.jsx)("br",{}),"— Yandex crowdsourcing experience"]})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part1.pdf",target:"_blank",children:"Part I: Main components of data collection via crowdsourcing"}),text:(0,u.jsxs)(u.Fragment,{children:["— Decomposition for an effective pipeline",(0,u.jsx)("br",{}),"— Task instruction & interface: best practices",(0,u.jsx)("br",{}),"— Quality control techniques"]})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part2.pdf",target:"_blank",children:"Part II: Introduction to Toloka for requesters"}),text:(0,u.jsxs)(u.Fragment,{children:["— How Toloka works",(0,u.jsx)("br",{}),"— Types of tasks in Toloka",(0,u.jsx)("br",{}),"— Creating a project in Toloka"]})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part3.pdf",target:"_blank",children:"Part III: Brainstorming the pipeline"}),text:(0,u.jsxs)(u.Fragment,{children:["— Dataset and required labels",(0,u.jsx)("br",{}),"— Discussion: how to collect labels?",(0,u.jsx)("br",{}),"— Data labeling pipeline for implementation"]})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part4.pdf",target:"_blank",children:"Part IV: Practical Session"}),text:(0,u.jsxs)(u.Fragment,{children:["Participants: ",(0,u.jsx)("br",{}),"— create ",(0,u.jsx)("br",{}),"— configure ",(0,u.jsx)("br",{}),"— run data labeling projects on real performers in real-time"]})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part5.pdf",target:"_blank",children:"Part V: Theory on efficient aggregation"}),text:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("br",{}),"— Aggregation models ",(0,u.jsx)("br",{})," — Incremental relabeling"," ",(0,u.jsx)("br",{})," — Dynamic pricing"]})},{date:"10:30 - 11:00",text:(0,u.jsx)("b",{children:"Break"})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part6.pdf",target:"_blank",children:"Part VI: Practical Session"}),text:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("br",{}),"— Completing the label collection process"]})},{date:(0,u.jsx)(d.r,{href:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part7.pdf",target:"_blank",children:"Part VII: Discussion of results and conclusions"}),text:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("br",{}),"— Project results",(0,u.jsx)("br",{}),"— Ideas for further work and research",(0,u.jsx)("br",{}),"— References to literature and other tutorials"]})}]}),(0,u.jsx)(n.w,{offset:56}),(0,u.jsx)(l.D,{level:2,styleAs:4,id:"slides",children:"Slides"}),(0,u.jsx)(a._,{files:[{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_introduction.pdf",title:"Introduction"},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part1.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 1:"})," Main components of data collection"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part2.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 2:"})," Toloka requester interface"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part3.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 3:"})," Brainstorming the pipeline"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part4.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 4:"})," Practical Session"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part5.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 5:"})," Theory on efficient aggregation"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part6.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 6:"})," Practical Session part 2"]})},{link:"https://tlk-infra-front.azureedge.net/portal-static/files/conference-presentations/sigmod/sigmod2020_part7.pdf",title:(0,u.jsxs)(u.Fragment,{children:[(0,u.jsx)("b",{children:"Part 7:"})," Results and conclusions"]})}]})]})})}}}]);