"use strict";(self.webpackChunk_toloka_portal_gatsby_renderer=self.webpackChunk_toloka_portal_gatsby_renderer||[]).push([[265],{31451:function(e,t,i){i.d(t,{YD:function(){return u}});var n=i(67294);function r(){return r=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var i=arguments[t];for(var n in i)Object.prototype.hasOwnProperty.call(i,n)&&(e[n]=i[n])}return e},r.apply(this,arguments)}var o=new Map,s=new WeakMap,a=0;function l(e){return Object.keys(e).sort().filter((function(t){return void 0!==e[t]})).map((function(t){return t+"_"+("root"===t?(i=e.root)?(s.has(i)||(a+=1,s.set(i,a.toString())),s.get(i)):"0":e[t]);var i})).toString()}function h(e,t,i){if(void 0===i&&(i={}),!e)return function(){};var n=function(e){var t=l(e),i=o.get(t);if(!i){var n,r=new Map,s=new IntersectionObserver((function(t){t.forEach((function(t){var i,o=t.isIntersecting&&n.some((function(e){return t.intersectionRatio>=e}));e.trackVisibility&&void 0===t.isVisible&&(t.isVisible=o),null==(i=r.get(t.target))||i.forEach((function(e){e(o,t)}))}))}),e);n=s.thresholds||(Array.isArray(e.threshold)?e.threshold:[e.threshold||0]),i={id:t,observer:s,elements:r},o.set(t,i)}return i}(i),r=n.id,s=n.observer,a=n.elements,h=a.get(e)||[];return a.has(e)||a.set(e,h),h.push(t),s.observe(e),function(){h.splice(h.indexOf(t),1),0===h.length&&(a.delete(e),s.unobserve(e)),0===a.size&&(s.disconnect(),o.delete(r))}}function c(e){return"function"!=typeof e.children}var d=function(e){var t,i;function o(t){var i;return(i=e.call(this,t)||this).node=null,i._unobserveCb=null,i.handleNode=function(e){i.node&&(i.unobserve(),e||i.props.triggerOnce||i.props.skip||i.setState({inView:!!i.props.initialInView,entry:void 0})),i.node=e||null,i.observeNode()},i.handleChange=function(e,t){e&&i.props.triggerOnce&&i.unobserve(),c(i.props)||i.setState({inView:e,entry:t}),i.props.onChange&&i.props.onChange(e,t)},i.state={inView:!!t.initialInView,entry:void 0},i}i=e,(t=o).prototype=Object.create(i.prototype),t.prototype.constructor=t,t.__proto__=i;var s=o.prototype;return s.componentDidUpdate=function(e){e.rootMargin===this.props.rootMargin&&e.root===this.props.root&&e.threshold===this.props.threshold&&e.skip===this.props.skip&&e.trackVisibility===this.props.trackVisibility&&e.delay===this.props.delay||(this.unobserve(),this.observeNode())},s.componentWillUnmount=function(){this.unobserve(),this.node=null},s.observeNode=function(){if(this.node&&!this.props.skip){var e=this.props,t=e.threshold,i=e.root,n=e.rootMargin,r=e.trackVisibility,o=e.delay;this._unobserveCb=h(this.node,this.handleChange,{threshold:t,root:i,rootMargin:n,trackVisibility:r,delay:o})}},s.unobserve=function(){this._unobserveCb&&(this._unobserveCb(),this._unobserveCb=null)},s.render=function(){if(!c(this.props)){var e=this.state,t=e.inView,i=e.entry;return this.props.children({inView:t,entry:i,ref:this.handleNode})}var o=this.props,s=o.children,a=o.as,l=o.tag,h=function(e,t){if(null==e)return{};var i,n,r={},o=Object.keys(e);for(n=0;n<o.length;n++)i=o[n],t.indexOf(i)>=0||(r[i]=e[i]);return r}(o,["children","as","tag","triggerOnce","threshold","root","rootMargin","onChange","skip","trackVisibility","delay","initialInView"]);return(0,n.createElement)(a||l||"div",r({ref:this.handleNode},h),s)},o}(n.Component);function u(e){var t=void 0===e?{}:e,i=t.threshold,r=t.delay,o=t.trackVisibility,s=t.rootMargin,a=t.root,l=t.triggerOnce,c=t.skip,d=t.initialInView,u=(0,n.useRef)(),f=(0,n.useState)({inView:!!d}),m=f[0],w=f[1],p=(0,n.useCallback)((function(e){void 0!==u.current&&(u.current(),u.current=void 0),c||e&&(u.current=h(e,(function(e,t){w({inView:e,entry:t}),t.isIntersecting&&l&&u.current&&(u.current(),u.current=void 0)}),{root:a,rootMargin:s,threshold:i,trackVisibility:o,delay:r}))}),[Array.isArray(i)?i.toString():i,a,s,l,c,o,r]);(0,n.useEffect)((function(){u.current||!m.entry||l||c||w({inView:!!d})}));var g=[p,m.inView,m.entry];return g.ref=g[0],g.inView=g[1],g.entry=g[2],g}d.displayName="InView",d.defaultProps={threshold:0,triggerOnce:!1,initialInView:!1}},92033:function(e,t,i){i.d(t,{J:function(){return q}});var n="bR_dN",r="bR_dP",o="bR_dQ",s="bR_bC",a="bR_dR",l="bR_dS",h="bR_dT",c="bR_dV",d="bR_dW",u="bR_dX",f="bR_dY",m="bR_dZ",w=i(79700),p=i.n(w),g={list:"d9_cl",item:"d9_3",button:"d9_6",activeItem:"d9_jB"},b=i(67294),v=i(55074),x=i(85893),j=function(e){var t=e.toc,i=(0,v.A)(t.map((function(e){return e.url})),t.length>0?t[0].url:void 0),n=(0,b.useCallback)((function(e){var t=document.getElementById(e.slice(1));if(t){var i=document.body.clientWidth>=1024?60:120,n=t.getBoundingClientRect();window.scrollTo({top:n.top+window.pageYOffset-n.height-i,behavior:"smooth"})}}),[]);return 0===t.length?(0,x.jsx)("div",{className:g.wrap}):(0,x.jsx)("div",{className:g.wrap,children:(0,x.jsx)("ul",{className:g.list,children:t.map((function(e){return(0,x.jsx)("li",{className:p()(g.item,e.url===i?g.activeItem:void 0),children:(0,x.jsx)("button",{className:p()("button-reset",g.button),onClick:function(){return n(e.url)},children:e.title})},e.url)}))})})},y=i(61123),k=i(4511),C=i(11467),_=i(38477),q=function(e){var t=e.children,i=e.toc,w=e.withoutShareLink,g=e.form,v=e.backLink,q=e.additionalLink,I=e.noBottomBorder,$=(0,b.useMemo)((function(){return g?(0,x.jsx)("div",{className:l,children:g}):null}),[g]);return(0,x.jsxs)("div",{className:f,children:[(0,x.jsxs)("div",{className:p()(o,i?void 0:m),children:[i&&i.length>0?(0,x.jsx)("div",{className:p()(n,I?r:void 0),children:(0,x.jsx)(j,{toc:i})}):null,w?null:(0,x.jsx)("div",{className:a,children:(0,x.jsx)(y.D,{})}),$]}),(0,x.jsxs)("div",{className:s,children:[(0,x.jsxs)(C.O,{children:[$,t]}),v||q?(0,x.jsxs)("div",{className:h,children:[v?(0,x.jsx)(k.r,{className:p()(c,"back-link"),onClick:function(){return(0,_.X)(v.href)},iconPosition:v.iconPosition,children:v.text}):null,q?(0,x.jsx)("div",{className:d,children:(0,x.jsx)(k.r,{href:q.href,className:u,iconPosition:q.iconPosition,children:q.text})}):null]}):null]})]})}},7291:function(e,t,i){i.d(t,{v:function(){return b}});var n=i(46237),r="bX_Z",o="bX_dK",s="bX_q",a="bX_h",l="bX_dS",h="bX_dd",c=i(99628),d=i(84401),u=i(19224),f=i(79700),m=i.n(f),w=i(14245),p=i(27850),g=i(85893),b=function(e){var t=e.title,i=e.text,f=e.inputPlaceholder,b=e.button,v=e.successText,x=e.sendingData,j=e.fractal,y=void 0===j?(0,g.jsx)(u.J,{}):j,k=(0,p.Q)(x),C=k.onSubmit,_=k.isSubmitted,q=k.setInputText;return(0,g.jsxs)("div",{className:m()(r,"light"),children:[t?(0,g.jsx)(n.D,{level:4,className:s,children:t}):null,(0,g.jsx)("p",{className:a,children:i}),(0,g.jsx)("form",{className:l,onSubmit:C,children:_?(0,g.jsx)(w.r,{text:v}):(0,g.jsxs)(g.Fragment,{children:[(0,g.jsx)(c.z,{view:"default",size:"m",type:"email",placeholder:f,className:o,onChange:function(e){return q(e.target.value)},autoComplete:"on",required:!0}),(0,g.jsx)(d.z,{view:b.view,type:"submit",size:b.size,children:b.text})]})}),(0,g.jsx)("div",{className:h,children:y})]})}},86457:function(e,t,i){i.d(t,{C:function(){return j}});var n=i(79700),r=i.n(n),o="bg_cV",s="bg_3",a="bg_cW",l="bg_cX",h="bg_v",c="bg_q",d="bg_bn",u="bg_cY",f="bg_j",m="bg_cZ",w="bg_b3",p="bg_c0",g="bg_c1",b=i(30759),v=i(4511),x=i(85893),j=function(e){var t=e.list,i=e.className,n=e.itemClassName,j=e.maxItemsInRow,y=e.isBigIcon,k=e.inLine,C=e.twoElementsInMobile,_=e.isSmallIcon;return(0,x.jsx)("ul",{className:r()("features-list-container",i,1===t.length||1===j?o:void 0,k?m:void 0,C?p:void 0,_?l:void 0),children:t.map((function(e,t){return(0,x.jsxs)("li",{className:r()(s,y?a:void 0,n,2===j?g:void 0,k?w:void 0,e.text||e.link||e.additional?void 0:h),children:[e.icon,(0,x.jsxs)("div",{children:[e.title?(0,x.jsx)(b.D,{level:5,className:c,children:e.title}):null,e.text?(0,x.jsx)("p",{className:d,children:e.text}):null,e.link?(0,x.jsx)("div",{className:f,children:(0,x.jsx)(v.r,{href:e.link.href,iconPosition:e.link.iconPosition,target:e.link.target,children:e.link.text})}):null,e.additional?(0,x.jsx)("div",{className:u,children:e.additional}):null]})]},t)}))})}},61123:function(e,t,i){i.d(t,{D:function(){return p}});var n={list:"fb_cl",item:"fb_3",button:"fb_6"},r=i(79700),o=i.n(r),s=i(67294),a=i(85893),l=function(){return(0,a.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"16",height:"13",viewBox:"0 0 16 13",fill:"none",children:(0,a.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M5.96907 12.861C5.27718 12.861 4.56503 12.8068 3.8486 12.6982C2.58677 12.5067 1.30944 11.8684 0.947991 11.6773L0 11.176L1.02636 10.8496C2.1483 10.4928 2.83079 10.2714 3.67564 9.92485C2.82969 9.52859 2.17772 8.8159 1.86388 7.89458L1.62511 7.19355L1.82091 7.22248C1.63524 7.04149 1.48864 6.85779 1.37499 6.69075C0.970818 6.09703 0.75683 5.37156 0.802362 4.74987L0.847284 4.13798L1.22606 4.27989C1.06628 3.98805 0.9508 3.67638 0.882928 3.34971C0.717524 2.55306 0.85595 1.7067 1.27282 0.966604L1.60277 0.380799L2.04405 0.893404C3.43967 2.51481 5.20736 3.47662 7.30536 3.75819C7.21967 3.18691 7.28388 2.63558 7.4964 2.14139C7.74384 1.56598 8.18402 1.07804 8.76898 0.73025C9.41867 0.344084 10.2065 0.156962 10.9874 0.203359C11.8158 0.252589 12.568 0.552809 13.1654 1.07214C13.4572 0.998835 13.6722 0.920559 13.9627 0.81478C14.1374 0.751146 14.3357 0.678895 14.5837 0.596254L15.4978 0.291666L14.9018 1.9381C14.9411 1.93491 14.9816 1.93219 15.0239 1.93043L16 1.88769L15.4231 2.65022C15.39 2.6939 15.3816 2.7063 15.3699 2.72342C15.3234 2.79118 15.2655 2.87547 14.4737 3.89809C14.2754 4.15416 14.1764 4.48767 14.1948 4.83736C14.2651 6.16621 14.0964 7.36851 13.6935 8.41073C13.3124 9.39676 12.7219 10.2427 11.9384 10.9252C10.9689 11.7696 9.73271 12.3477 8.26397 12.6433C7.54352 12.7883 6.76985 12.861 5.96907 12.861Z",fill:"#1E2126",fillOpacity:"0.5"})})},h=function(){return(0,a.jsxs)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"18",height:"18",viewBox:"0 0 18 18",fill:"none",children:[(0,a.jsx)("path",{d:"m9 14.14-1.29 1.29a3.64 3.64 0 0 1-5.14 0v0a3.64 3.64 0 0 1 0-5.14L5.14 7.7a3.64 3.64 0 0 1 5.15 0v0l.28.29",stroke:"#1E2126",strokeOpacity:".5",strokeWidth:"2.4",strokeLinecap:"round",strokeLinejoin:"round"}),(0,a.jsx)("path",{d:"m9 3.86 1.29-1.29a3.64 3.64 0 0 1 5.14 0v0a3.64 3.64 0 0 1 0 5.14l-2.57 2.58a3.64 3.64 0 0 1-5.15 0v0l-.21-.22",stroke:"#1E2126",strokeOpacity:".5",strokeWidth:"2.4",strokeLinecap:"round",strokeLinejoin:"round"})]})},c=function(){return(0,a.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"8",height:"16",viewBox:"0 0 8 16",fill:"none",children:(0,a.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M6.54 2.65673H8.00067V0.112728C7.74867 0.078061 6.882 6.10352e-05 5.87267 6.10352e-05C3.76667 6.10352e-05 2.324 1.32473 2.324 3.75939V6.00006H0V8.84406H2.324V16.0001H5.17333V8.84473H7.40333L7.75733 6.00073H5.17267V4.04139C5.17333 3.21939 5.39467 2.65673 6.54 2.65673Z",fill:"#1E2126",fillOpacity:"0.5"})})},d=function(){return(0,a.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"16",height:"16",viewBox:"0 0 16 16",fill:"none",children:(0,a.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M2.15471 0.207764C0.966598 0.207764 0 1.15446 0 2.31809C0 3.48225 0.966598 4.4293 2.15471 4.4293C3.34188 4.4293 4.30769 3.4822 4.30769 2.31809C4.30775 1.15446 3.34188 0.207764 2.15471 0.207764ZM3.43672 5.03238H0.255583C0.114399 5.03238 0 5.14816 0 5.29093V15.6292C0 15.772 0.114399 15.8878 0.255583 15.8878H3.43672C3.57791 15.8878 3.69231 15.772 3.69231 15.6292V5.29093C3.69231 5.14816 3.57791 5.03238 3.43672 5.03238ZM9.11072 6.19188C9.68593 5.57515 10.6343 5.03238 11.9172 5.03238C15.5149 5.03238 16 7.59188 16 10.0956V15.6351C16 15.7747 15.8852 15.8878 15.7433 15.8878H12.555C12.4132 15.8878 12.2984 15.7747 12.2984 15.6351V10.725C12.2984 9.35023 12.1951 8.29862 10.8963 8.29862C9.70362 8.29862 9.23894 8.9546 9.23894 10.6383V15.6351C9.23894 15.7746 9.12407 15.8877 8.98231 15.8877H5.7951C5.65333 15.8877 5.53847 15.7746 5.53847 15.6351V5.53596C5.53847 5.39649 5.65333 5.28338 5.7951 5.28338H8.85409C8.99586 5.28338 9.11072 5.39649 9.11072 5.53596V6.19188Z",fill:"#1E2126",fillOpacity:"0.5"})})},u=i(9546),f=i(47250),m=i(81547);function w(e){var t=[];for(var i in e)t.push(encodeURIComponent(i)+"="+encodeURIComponent(e[i]));return t.join("&")}var p=function(){if(m.j){var e=window.location.href,t=(0,u.F)(e),i=(0,s.useMemo)((function(){var t={url:e,utm_source:"share2"},i={u:e,src:"sp",utm_source:"share2"},n={mini:"true",url:e,utm_source:"share2"};return[{icon:(0,a.jsx)(l,{}),title:"Twitter",link:"https://twitter.com/intent/tweet?"+w(t)},{icon:(0,a.jsx)(c,{}),title:"Facebook",link:"https://www.facebook.com/sharer.php?"+w(i)},{icon:(0,a.jsx)(d,{}),title:"Linkedin",link:"https://www.linkedin.com/shareArticle?"+w(n)}]}),[e]);return(0,a.jsx)("div",{className:n.wrap,children:(0,a.jsxs)("ul",{className:n.list,children:[i.map((function(e){return(0,a.jsx)("li",{className:n.item,children:(0,a.jsxs)("a",{className:o()("button-reset",n.button),href:e.link,target:"_blank",rel:"nofollow noopener noreferrer",title:e.title,children:[e.icon,(0,a.jsx)(f.M,{children:e.title})]})},e.title)})),(0,a.jsx)("li",{className:n.item,children:(0,a.jsxs)("a",{className:o()("button-reset",n.button),href:"#",target:"_blank",rel:"nofollow noopener noreferrer",title:"Link to current page",onClick:function(e){return e.preventDefault()},ref:t,children:[(0,a.jsx)(h,{}),(0,a.jsx)(f.M,{children:"Link to current page"})]})})]})})}return null}},33543:function(e,t,i){i.d(t,{M:function(){return h}});var n={tab:"bW_cd",sliderWrap:"bW_cc",noSidePaddings:"bW_cn",defaultView:"bW_d5",dark:"bW_c3",modifiedView:"bW_d6"},r=i(79700),o=i.n(r),s=i(67294),a=i(21648),l=i(85893),h=function(e){var t=e.items,i=e.render,r=e.shouldTabsChangeView,h=void 0===r||r,c=e.theme,d=e.noSidePaddings,u=(0,a.u)(),f=u.ref,m=u.setSwiperRef,w=(0,s.useState)(0),p=w[0],g=w[1];return(0,l.jsxs)(l.Fragment,{children:[t.length>1?(0,l.jsx)("div",{className:o()(n.tab,c?n[c]:void 0,"tab-slider",h?n.modifiedView:n.defaultView,d?n.noSidePaddings:void 0),ref:f,children:t.map((function(e,t){return(0,l.jsxs)("button",{className:t===p?"active":void 0,onClick:function(){return g(t)},children:[e.icon,(0,l.jsx)("span",{children:e.title})]},t)}))}):null,(0,l.jsx)("div",{className:o()(n.sliderWrap,t.length>1?"withTabs":void 0),children:i({activeTab:p,setSwiperRef:m})})]})}},55074:function(e,t,i){i.d(t,{A:function(){return r}});var n=i(67294),r=function(e,t){var i=(0,n.useState)(t),r=i[0],o=i[1];return(0,n.useEffect)((function(){var t=document.body.clientWidth>=1024?100:150,i=new IntersectionObserver((function(e){e.forEach((function(e){!1!==e.isIntersecting&&e.boundingClientRect.y<200&&o("#"+e.target.id)}))}),{root:null,rootMargin:"-"+t+"px 0px 0px 0px",threshold:[0,.1,.2,.3,.4,.5,.6,.7,.8,.9,1]});return e.map((function(e){return document.getElementById(e.slice(1))})).filter(Boolean).forEach((function(e){i.observe(e)})),function(){i.disconnect()}}),[e,t]),r}},21648:function(e,t,i){i.d(t,{u:function(){return o}});var n=i(67294),r=i(31451),o=function(){var e=(0,n.useState)(null),t=e[0],i=e[1],o=(0,r.YD)({threshold:0,triggerOnce:!0}),s=o.ref,a=o.inView;return(0,n.useEffect)((function(){t&&(t.autoplay.stop(),a&&t.autoplay.start())}),[a,t]),{ref:s,swiperRef:t,setSwiperRef:i}}},38477:function(e,t,i){i.d(t,{X:function(){return n}});var n=function(e){document.referrer.includes(e)?window.history.back():window.open(e,"_self")}},67911:function(e,t,i){i.d(t,{A:function(){return v}});var n="F_Z",r="F_b2",o="F_b3",s="F_b4",a="F_b5",l="F_b6",h="F_b7",c="F_b8",d="F_b9",u="F_cb",f=i(79700),m=i.n(f),w=i(86457),p=i(67294),g=i(85893),b={1:u,2:c,3:d,4:void 0},v=function(e){var t=e.list,i=e.maxItemsInRow,c=void 0===i?4:i,d=e.isBigIcon,u=e.withImage,f=e.inLine,v=e.twoElementsInMobile,x=e.columnFlow,j=e.paddingsBeforeDesktop,y=e.isSmallIcon,k=(0,p.useRef)(null);return(0,p.useEffect)((function(){k.current&&x&&k.current.style.setProperty("--count",""+Math.ceil(t.length/c))}),[k]),(0,g.jsx)("div",{ref:k,className:m()(n,f?o:void 0,j?r:""),children:(0,g.jsx)(w.C,{list:t,className:m()(s,b[c],u?a:void 0,x&&4!==c?h:void 0),itemClassName:u?l:void 0,maxItemsInRow:c,isBigIcon:d,inLine:f,twoElementsInMobile:v,isSmallIcon:y})})}},66924:function(e,t,i){i.d(t,{n:function(){return r}});var n=i(85893),r=function(){return(0,n.jsxs)("svg",{xmlns:"http://www.w3.org/2000/svg",width:"24",height:"25",viewBox:"0 0 24 25",fill:"none",children:[(0,n.jsx)("g",{clipPath:"url(#clip0_9129_114992)",children:(0,n.jsx)("path",{fillRule:"evenodd",clipRule:"evenodd",d:"M4.92861 19.5714C1.02365 15.6664 1.02373 9.33478 4.92926 5.42926C8.83478 1.52373 15.1664 1.52365 19.0714 5.42861C22.9764 9.33357 22.9763 15.6652 19.0707 19.5707C15.1652 23.4763 8.83357 23.4764 4.92861 19.5714ZM3.51439 20.9856C-1.17172 16.2995 -1.17143 8.70152 3.51504 4.01504C8.20152 -0.671427 15.7995 -0.671719 20.4856 4.01439C25.1717 8.7005 25.1714 16.2985 20.485 20.985C15.7985 25.6714 8.2005 25.6717 3.51439 20.9856ZM11.6703 16.0403C11.4839 16.2396 11.2243 16.354 10.9515 16.3571C10.6787 16.3602 10.4165 16.2518 10.2256 16.0569L6.99992 12.7633C6.61349 12.3687 6.62008 11.7356 7.01465 11.3492C7.40922 10.9627 8.04235 10.9693 8.42879 11.3639L10.9235 13.9111L15.5555 8.95973C15.9328 8.55641 16.5656 8.53533 16.9689 8.91263C17.3723 9.28993 17.3933 9.92274 17.016 10.3261L11.6703 16.0403Z",fill:"#0055D9"})}),(0,n.jsx)("defs",{children:(0,n.jsx)("clipPath",{id:"clip0_9129_114992",children:(0,n.jsx)("rect",{width:"24",height:"24",fill:"white",transform:"translate(0 0.5)"})})})]})}},88909:function(e,t,i){i.r(t),i.d(t,{default:function(){return j}});var n=i(74533),r=i(88586),o=i(37990),s=i(52893),a=i(93252),l=i(51512),h=i(25109),c=i(7291),d=i(92033),u=i(46237),f=i(38100),m=i(11467),w=i(88709),p=i(33543),g=i(4511),b=i(67911),v=i(66924),x=i(85893);function j(){return(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)(s.q,{og:{img:"https://tlk-infra-front.azureedge.net/portal-static/images/knowledgebase/aggregation/og/share-img.jpg",title:"Aggregation Methods"}}),(0,x.jsx)(n.n,{}),(0,x.jsxs)(r.i,{headerPadding:!0,children:[(0,x.jsx)(o.h,{defaultBackground:"dark"}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsxs)(h.L,{overflow:"inherit",children:[(0,x.jsx)(u.D,{level:1,children:"Aggregation Methods"}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsxs)(d.J,{withoutShareLink:!0,noBottomBorder:!0,toc:[{url:"#example",title:"Example"},{url:"#methods",title:"Methods"}],form:(0,x.jsx)(c.v,{text:"Subscribe to Toloka News",inputPlaceholder:"Enter your email",button:{view:"action",text:"Subscribe",size:"l"},sendingData:{address:"/api/sender/subscribe",listId:"0ASPRWP3-H6V"},successText:"Thank you for subscribing"}),children:[(0,x.jsx)("p",{children:"In recent years crowdsourcing has become a standard tool for labeling massive datasets. It offers an appealing alternative to expensive expert labelers and distributes the task across a large crowd of low-cost non-expert workers. The power of crowdsourcing rests on the wisdom of the crowd — it has been observed in many contexts that judiciously aggregated independent opinions of unskilled individuals can outperform judgments of domain experts. Thus, methods for aggregating and drawing meaningful inferences from the noisy labels obtained from crowd workers are the cornerstone of crowdsourcing. In this article, we will discuss the foundations of aggregation techniques."}),(0,x.jsx)(l.w,{offset:24}),(0,x.jsx)(u.D,{level:2,styleAs:4,id:"example",children:"Example"}),(0,x.jsx)("p",{children:"Before we delve into details, let us introduce a running example that we will use throughout the article. Suppose we have a dataset of images and our goal is to classify each image as a cat or a dog. The crowdsourcing paradigm prescribes to distribute the task across multiple workers, each of whom labels a subset of the images. As a result, for each image we have several labels and the goal of the aggregation method is to combine multiple noisy labels into a single high-quality answer to the task. For brevity, in this article we will consider the binary classification task and give references to the literature where the multiclass case is described."}),(0,x.jsx)(m.V.Image,{imageId:"xE8hV_5A6ySbZlF7fb3m"}),(0,x.jsx)(l.w,{offset:24}),(0,x.jsx)(u.D,{level:2,styleAs:4,id:"methods",children:"Methods"}),(0,x.jsx)(p.M,{noSidePaddings:!0,shouldTabsChangeView:!1,items:[{title:"Majority vote"},{title:"Incorporating worker abilities"},{title:"Incorporating question difficulty"}],render:function(e){var t=e.activeTab;return[(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)(u.D,{level:5,children:"Method"}),(0,x.jsx)("p",{children:"Let's begin by introducing a simple and intuitive tool that is often used in practice: the majority vote aggregation algorithm. The idea of majority vote is to follow the most popular opinion among the workers. Majority Vote: the most popular label is set as the aggregated answer. Majority vote is a simple yet practical tool — it rests on the assumption that most of the workers are more likely to select the correct answer, and it compensates for noise by using the redundancy of the obtained labels."}),(0,x.jsx)(m.V.Image,{imageId:"bFI2HoYgPJqOhS7XWSTC"}),(0,x.jsx)("p",{children:"In fact, majority vote often achieves decent results and is used in many real-life projects. It also has the benefit of being very intuitive and practitioners can easily understand the reasoning behind the method. However, in crowdsourcing we operate under time and budget constraints. Hence, our goal is to collect as little data as possible to achieve the desired accuracy. From this perspective, majority vote may not always be the optimal choice. To understand the weaknesses of the method, let's look at the model behind majority vote."}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"Model"}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"The model behind the method is simple. There are $\\text{N}$ images and $\\text{M}$ workers. Each image $j \\in \\{1,...,N\\}$ has an unknown answer (a cat or a dog in our case). Majority vote assumes that if worker $i$ labels image $j$, then the worker gives the correct answer with some probability $p>1/2$:"})}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"$\\mathbb{P}[\\text{Worker } i \\text{ answers question } j \\text{ correctly}]=p$"})}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Note that the probability of correctness is assumed to be the same for each worker and for each question. The assumption of $p>1/2$ ensures that each worker is more likely to give the correct answer than the incorrect answer. Hence, if the number of labels for each image is large enough, majority vote will uncover the true answers with high probability."})}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"Limitations"}),(0,x.jsx)("p",{children:"The limitations of majority vote lie in its simplicity:"}),(0,x.jsx)(b.A,{inLine:!0,maxItemsInRow:2,list:[{icon:(0,x.jsx)(v.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:"Homogeneity of workers."})," First, majority vote assumes that workers all have the same ability. In other words, for a given question, the probability that a worker answers the question correctly is the same for all workers. However, performers on crowdsourcing platforms are very diverse — some of them are very careful while others can make careless mistakes. Thus, one direction for improvement over majority vote is to accommodate the variation in performer ability in the model."]})},{icon:(0,x.jsx)(v.n,{}),additional:(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("b",{children:"Homogeneity of questions."})," Second, majority vote assumes that questions are of the same difficulty. In other words, for a given worker, the probability that the worker answers a question correctly is the same for all questions. However, some questions within a project may be more difficult than others. Thus, another direction for improvement over majority vote is to accommodate the variation in question difficulty in the model."]})}]}),(0,x.jsx)("span",{children:"Let's look at both directions for improvement and examine some more advanced algorithms that are designed to capture the nature of crowdsourcing tasks."})]}),(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)(u.D,{level:5,children:"Model"}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"A natural way to incorporate the varying abilities of performers is to introduce a quality parameter for each worker. If there are workers $\\text{M}$, then we can associate each worker $j \\in \\{1,...,M\\}$ with an unknown quality parameter $p_{i} \\in [0,1]$. The higher the quality of the performer, the higher the probability that the performer answers a question correctly:"})}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"$\\mathbb{P}[\\text{Worker } i \\text{ answers question } j \\text{ correctly}]=p_{i}$"})}),(0,x.jsx)("p",{children:"In other words, the probability that a worker answers a question correctly varies across workers, but it is still independent of the question."}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"Methods"}),(0,x.jsx)("p",{children:"Given that workers have different abilities, it makes sense to give more weight to answers from stronger workers and less weight to answers from weaker workers. However, the problem is that we have to discover the quality of each worker. The main idea of the two methods we discuss in this section is to estimate both quality parameters of the workers and the answers to the questions."}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"When we have many golden set questions"}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Golden set questions, or control tasks, are questions for which the requester knows the correct answers in advance. In practice, we often add a certain number of golden set questions to the dataset to control the quality of the workers. When we have many golden set questions in the data, we can use them to estimate worker quality. Indeed, suppose that we have $G$ golden set questions and consider performer i who answers $k_{i}$ out of $G$ golden set questions correctly. Then we can set their estimated quality parameter to:"})}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"$\\widehat{p}_{i}=\\frac{k_{i}}{G}$"})}),(0,x.jsxs)("p",{children:["Now that we have estimates of the quality parameters, we can weigh the answer of each worker differently to arrive at the aggregated answers. This idea brings us to the",(0,x.jsx)("i",{children:"weighted majority vote"}),"."]}),(0,x.jsxs)("p",{children:[(0,x.jsx)("b",{children:"Weighted Majority Vote:"})," each label is weighted by the estimated quality of the performer."]}),(0,x.jsx)("p",{children:"The idea behind this method is illustrated in the figure below. Suppose that we have a tricky image in which a dog looks like a cat. In this case, majority vote treats the answers of low-ability workers (the first two workers) similarly to that of an expert worker (the last worker) and hence makes an error. In contrast, weighted majority vote additionally weighs each answer by the estimated quality of the worker. It arrives at the correct answer eventually, because the expert worker in this case outweighs the opinions of the other two workers."}),(0,x.jsx)(m.V.Image,{imageId:"5dqTbEnjeqaasO4H9rDK"}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"When we don't have many golden set questions"}),(0,x.jsx)("p",{children:"Weighted majority vote is a good option when we have enough golden set questions to estimate worker quality. However, in practice we often do not have many golden set questions and our estimates can be quite noisy. Additionally, workers may collectively discover what questions are in the golden set and cheat by providing correct answers to the golden set questions and random answers to the other questions. In this case, the Dawid-Skene method can help us learn the quality parameters of performers from answers with unknown questions:"}),(0,x.jsxs)("p",{children:[(0,x.jsx)("b",{children:"Dawid-Skene method:"})," simultaneously find the values of the performers' quality and the answers to the questions that agree with the observed data the most."]}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Let's take a closer look at the idea of the Dawid-Skene method in a simple example. Suppose that we have just $N=4$ questions and $M=3$ workers and each worker answers all the questions. In this case, the observed data is simply the responses of the workers to the questions."})}),(0,x.jsx)(m.V.Image,{imageId:"SWOwfLLsxAOxLN3dX5qc",size:"75%",justify:"centre"}),(0,x.jsx)("p",{children:"To understand how the Dawid-Skene method finds performer quality and answers to the questions that fit the data well, let's consider two options depicted in the image below. Each image suggests a set of parameters and we now try to understand which of the proposed options fits the data better."}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Let's take a closer look at the idea of the Dawid-Skene method in a simple example. Suppose that we have just $N=4$ questions and $M=3$ workers and each worker answers all the questions. In this case, the observed data is simply the responses of the workers to the questions."})}),(0,x.jsx)(m.V.Image,{imageId:"-c73EJK-wcOL3O5XPqyS"}),(0,x.jsx)(m.V.Image,{imageId:"DbvXLQ6jDoIEm2UIbT8C"}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"First, observe that in both images the proposed answers agree with the answers of the worker who is estimated to have a high quality parameter. But what choice of parameters fits the data best? To answer this question, observe that the second and third workers are in perfect agreement. If these workers have quality parameters like in the first image $\\widehat{p}_{2}=\\widehat{p}_{3}=0.5$ then according to the model these workers should answer at random. Thus, a high amount of agreement between the workers appears to be surprising, because random workers are expected to disagree in every other question. In contrast, if workers 2 and 3 are experts like in the second image $(\\widehat{p}_{2}=\\widehat{p}_{3}=1)$, then we would expect them to have a high amount of agreement, and that is what we observe in the data. Intuitively, the second set of parameters agrees with the observed data better."})}),(0,x.jsxs)("p",{children:["The simple example we introduced illustrates that the concept of agreement between the candidate parameters and observed data enables us to rule out options that don't fit the data well. A more formal introduction of the Dawid-Skene method is given in a"," ",(0,x.jsx)(g.r,{href:"https://www.jstor.org/stable/2346806?seq=1",target:"_blank",children:"seminal paper by Dawid and Skene"}),"."]}),(0,x.jsxs)("span",{children:["Both Weighted Majority Vote and David-Skene aggregation methods are an off-the-shelf feature in Toloka. To learn more about getting aggregated results from a labeled pool, see our"," ",(0,x.jsx)(g.r,{href:"https://yandex.ru/support/toloka-requester/concepts/result-aggregation.html?lang=en",target:"_blank",children:"documentation"}),"."]})]}),(0,x.jsxs)(x.Fragment,{children:[(0,x.jsx)("p",{children:"In this section, we show how to account for heterogeneity of performers when aggregating their answers. We begin with formulating the model and then discuss specific methods that are designed for this model."}),(0,x.jsx)("p",{children:"In the previous section we demonstrated how the diversity of workers' abilities can be incorporated into the model to make it more reflective of the real crowdsourcing setup. The Weighted Majority Vote and Dawid-Scene aggregation methods are the foundation of modern crowdsourcing and many requesters boost the quality of aggregated data by employing these methods. In this section, we continue the path of generalization and give a brief overview of modern approaches that additionally aim to account for question difficulty when aggregating the answers. The field of crowdsourcing is rapidly developing, and new methods that better capture the real situation are offered every year. Therefore, in this section we do not introduce concrete methods and instead outline different modeling approaches. We also give references to the literature where more details on specific implementations are provided."}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"Parametric approach"}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Similar to how we captured the quality of each worker by introducing a quality parameter $p_{i}$ for each worker, we can also introduce a difficulty parameter $d_{j}$ for each question. However, the main challenge is to describe how an interplay between the performer quality and question difficulty determine the probability that a given performer answers a given question correctly. A work by Whitehill et al., 2009 suggests the following solution."})}),(0,x.jsx)(b.A,{inLine:!0,maxItemsInRow:2,list:[{icon:(0,x.jsx)(v.n,{}),additional:(0,x.jsx)(f.S,{withoutContainer:!0,content:"First, it extends the range of worker quality from the $[0,1]$ quality range range we had above to $(-\\infty, \\infty)$. In particular, zero quality $p=0$ means that the worker answers all the questions at random. Positive values of quality imply that a worker is more likely to give a correct answer and negative values mean that the worker is adversarial and is more likely to give an incorrect label."})},{icon:(0,x.jsx)(v.n,{}),additional:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Second, the difficulty of each question $d \\in (0,\\infty)$ also has an intuitive interpretation: low difficulty of a question $(d\\approx0)$ means that the question is so easy that any worker answers it correctly with probability close to 1. The higher the value of the difficulty, the less likely it is that a given performer will answer the question correctly."})}]}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Putting these parameters together, the model assumes that the probability of a given worker $i$ answering a given question $j$ correctly is assumed to be described by the parametric expression:"})}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"$\\mathbb{P}[\\text{Worker } i \\text{ answers question } j \\text{ correctly}]=\\frac{1}{\\text{1exp}\\{-\\frac{p_i}{d_j}\\}}$"})}),(0,x.jsxs)("p",{children:["Note that in this case, the probability is function of both the worker and the question.",(0,x.jsx)("br",{})," Once we assume the specific parametric equation to connect the performer quality and question difficulty to the probability of a correct answer, we can employ the same idea we used to learn parameters under the Dawid-Skene model. In this way, we can get the estimates of both the model parameters and the answers to the questions. For more details, please refer to"," ",(0,x.jsx)(g.r,{href:"https://arxiv.org/pdf/1602.03481.pdf",target:"_blank",children:"the paper by Ashish Khetan and Sewoong Oh"}),"."]}),(0,x.jsx)("p",{children:"While parametric models enable efficient inference, they unavoidably make strong assumptions on the cognitive processes invoked when workers answer the questions. These assumptions are typically non-testable and it is unclear how well they agree with reality. If the assumptions implied by the parametric model are wrong, the methods that work under these models may lead to unexpected results. This caveat brings us to the idea of non-parametric approaches, which try to avoid strong assumptions on how people think."}),(0,x.jsx)(l.w,{offset:8}),(0,x.jsx)(m.V.Line,{}),(0,x.jsx)(l.w,{offset:32}),(0,x.jsx)(u.D,{level:5,children:"Non-parametric approach"}),(0,x.jsx)("p",{children:(0,x.jsx)(f.S,{withoutContainer:!0,content:"A non-parametric approach proposed by Shah et al., 2016 refrains from modeling probabilities that worker $i$ answers question $j$ correctly. Instead, it restricts the relationship between these probabilities. Specifically, there are two key assumptions:"})}),(0,x.jsx)(b.A,{inLine:!0,maxItemsInRow:2,list:[{icon:(0,x.jsx)(v.n,{}),additional:(0,x.jsx)(f.S,{withoutContainer:!0,content:"First, it assumes that workers can be ordered in terms of their ability. If worker $i_1$ is higher in this ordering than worker $i_2$, then for each question worker $i_1$ has a higher probability of being correct than worker $i_2$."})},{icon:(0,x.jsx)(v.n,{}),additional:(0,x.jsx)(f.S,{withoutContainer:!0,content:"Second, it assumes that questions can be ordered in terms of their difficulty. If question $j_1$ is more difficult than question $j_2$, then any worker makes a mistake in question $j_1$ with at least as high probability as in question $j_2$."})}]}),(0,x.jsxs)("p",{children:["Observe that these assumptions are much weaker than those made by the parametric model. Indeed, the parametric model not only induces the existence of such orderings, but also models all the probabilities. In contrast, the non-parametric approach only makes a natural assumption on the orderings but does not restrict the specific cognitive mechanisms taking place in workers' minds. It has been demonstrated that in some cases the non-parametric model allows for an efficient inference. For more details, see"," ",(0,x.jsx)(g.r,{href:"https://arxiv.org/pdf/1606.09632.pdf",target:"_blank",children:"the full paper"}),"."]}),(0,x.jsx)("p",{children:"As we have noted, the approaches we discussed in this section are more novel and are not currently used as standard approaches in crowdsourcing. If you think that the question difficulty varies significantly in your project, then we encourage you to study the referenced methods and the underlying assumptions more carefully and try them out!"})]})][t]}})]})]}),(0,x.jsx)(l.w,{offset:40}),(0,x.jsx)(a.$,{}),(0,x.jsx)(w.f,{})]})]})}}}]);