{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":219035799,"defaultBranch":"main","name":"tokenizers","ownerLogin":"huggingface","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2019-11-01T17:52:20.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/25720743?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1717420920.0","currentOid":""},"activityList":{"items":[{"before":"fa16167fbae87d3ab171c743fe8e6e865ba010bf","after":"d998a7a1477a56b52b7cb93f61e99040c6bf1134","ref":"refs/heads/add-display","pushedAt":"2024-06-04T14:31:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"nit","shortMessageHtmlLink":"nit"}},{"before":"9621b6213f4f688940d3ff4ff3ea7698774cf1cc","after":"fa16167fbae87d3ab171c743fe8e6e865ba010bf","ref":"refs/heads/add-display","pushedAt":"2024-06-04T14:22:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"more nits","shortMessageHtmlLink":"more nits"}},{"before":"fefafa71b6aacc3305205d59709967d326cd43c4","after":"9621b6213f4f688940d3ff4ff3ea7698774cf1cc","ref":"refs/heads/add-display","pushedAt":"2024-06-04T14:16:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"updates","shortMessageHtmlLink":"updates"}},{"before":"20c9fc4fe8d90911d3c9437d306689bcf673d4f2","after":"fefafa71b6aacc3305205d59709967d326cd43c4","ref":"refs/heads/add-display","pushedAt":"2024-06-04T14:07:47.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"update","shortMessageHtmlLink":"update"}},{"before":"fda66f59f6e935d64f8b21d454bef379d0907c0c","after":"20c9fc4fe8d90911d3c9437d306689bcf673d4f2","ref":"refs/heads/add-display","pushedAt":"2024-06-04T12:37:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"updates","shortMessageHtmlLink":"updates"}},{"before":"6413810b6e961f4740ea0471c7ea4b049859e424","after":"fda66f59f6e935d64f8b21d454bef379d0907c0c","ref":"refs/heads/add-display","pushedAt":"2024-06-04T12:29:48.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"nits","shortMessageHtmlLink":"nits"}},{"before":"904ce70a3c85ac46431d0ceb330b7b9573b67a1d","after":"6413810b6e961f4740ea0471c7ea4b049859e424","ref":"refs/heads/add-display","pushedAt":"2024-06-04T12:10:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"finallllly!","shortMessageHtmlLink":"finallllly!"}},{"before":"4c2aca1e96050d787f1611fe743d2e7f7ca445cf","after":"904ce70a3c85ac46431d0ceb330b7b9573b67a1d","ref":"refs/heads/add-display","pushedAt":"2024-06-04T11:22:21.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"this kinda works","shortMessageHtmlLink":"this kinda works"}},{"before":"22242755dc2958ff2fbf56a39975bea5ba368b89","after":"4c2aca1e96050d787f1611fe743d2e7f7ca445cf","ref":"refs/heads/add-display","pushedAt":"2024-06-04T08:15:25.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"working shit","shortMessageHtmlLink":"working shit"}},{"before":"a90ec224c42720fcfa69a06a79600a0c8d6b4731","after":"22242755dc2958ff2fbf56a39975bea5ba368b89","ref":"refs/heads/add-display","pushedAt":"2024-06-04T06:02:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"current modifications","shortMessageHtmlLink":"current modifications"}},{"before":"f4af6162a8ba84c3d229924ec97c95d39402fc54","after":"a90ec224c42720fcfa69a06a79600a0c8d6b4731","ref":"refs/heads/add-display","pushedAt":"2024-06-03T16:41:46.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"does not work","shortMessageHtmlLink":"does not work"}},{"before":"f1a6a97c821567726f120bed409bd00d461f3c4f","after":"f4af6162a8ba84c3d229924ec97c95d39402fc54","ref":"refs/heads/add-display","pushedAt":"2024-06-03T16:14:59.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"update","shortMessageHtmlLink":"update"}},{"before":"61804d9b73dc3f28621c141f89d7cc2be6a1c9ce","after":"f1a6a97c821567726f120bed409bd00d461f3c4f","ref":"refs/heads/add-display","pushedAt":"2024-06-03T14:08:30.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"make it work for the model for now","shortMessageHtmlLink":"make it work for the model for now"}},{"before":null,"after":"61804d9b73dc3f28621c141f89d7cc2be6a1c9ce","ref":"refs/heads/add-display","pushedAt":"2024-06-03T13:22:00.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"initial commit","shortMessageHtmlLink":"initial commit"}},{"before":"4992d242a5f4793211a83646ee479b9eb8d6b705","after":null,"ref":"refs/heads/break_decoder_added_tokens","pushedAt":"2024-05-06T09:49:39.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/204321?s=80&v=4"}},{"before":"f2ec3b239b0a7a9866b01ec5cbd4d44243a40a16","after":"25aee8b88c8de3c5a52e2f9cb6281d6df00ad516","ref":"refs/heads/main","pushedAt":"2024-05-06T09:49:38.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/204321?s=80&v=4"},"commit":{"message":"[BREAKING CHANGE] Ignore added_tokens (both special and not) in the decoder (#1513)\n\n* [BREAKING CHANGE] Ignore added_tokens (both special and not) in the\r\ndecoder\r\n\r\nCauses issues with `ByteLevel` messing up some `AddedTokens` with some\r\nutf-8 range used in the bytelevel mapping.\r\n\r\nThis commit tests the extend of the damage of ignoring the decoder for\r\nthose tokens.\r\n\r\n* Format.\r\n\r\n* Installing cargo audit.\r\n\r\n* Minor fix.\r\n\r\n* Fixing \"bug\" in node/python.\r\n\r\n* Autoformat.\r\n\r\n* Clippy.\r\n\r\n* Only prefix space when there's no decoder.","shortMessageHtmlLink":"[BREAKING CHANGE] Ignore added_tokens (both special and not) in the d…"}},{"before":"443e5333d6dcea7f8dda5859d435c0994fca831b","after":null,"ref":"refs/heads/dependabot/npm_and_yarn/tokenizers/examples/unstable_wasm/www/express-4.19.2","pushedAt":"2024-05-04T01:49:10.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"dependabot[bot]","name":null,"path":"/apps/dependabot","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/29110?s=80&v=4"}},{"before":"71c2a8d01a56cd7bd28148c309e210c47dac78e7","after":"f2ec3b239b0a7a9866b01ec5cbd4d44243a40a16","ref":"refs/heads/main","pushedAt":"2024-04-30T13:53:47.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"remove enforcement of non special when adding tokens (#1521)\n\n* remove enforcement of non special when adding tokens\r\n\r\n* mut no longer needed\r\n\r\n* add a small test\r\n\r\n* nit\r\n\r\n* style\r\n\r\n* audit\r\n\r\n* ignore cargo audit's own vulnerability\r\n\r\n* update\r\n\r\n* revert\r\n\r\n* remove CVE","shortMessageHtmlLink":"remove enforcement of non special when adding tokens (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2270901030\" data-permission-text=\"Title is private\" data-url=\"https://github.com/huggingface/tokenizers/issues/1521\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/huggingface/tokenizers/pull/1521/hovercard\" href=\"https://github.com/huggingface/tokenizers/pull/1521\">#1521</a>)"}},{"before":"3579eb61112cd6c89d6a9e72ae97b3dd853ddf1f","after":null,"ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T13:53:47.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"}},{"before":"7b8d4e68398fa11669eb39c964b85422b8dc7398","after":"3579eb61112cd6c89d6a9e72ae97b3dd853ddf1f","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T13:46:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"remove CVE","shortMessageHtmlLink":"remove CVE"}},{"before":"d70ba4236c806fff002d45aa3c42ec06004cd9bf","after":"7b8d4e68398fa11669eb39c964b85422b8dc7398","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T13:45:26.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"revert","shortMessageHtmlLink":"revert"}},{"before":"65b495895ff411e7a208a368796a11624771bfaf","after":"d70ba4236c806fff002d45aa3c42ec06004cd9bf","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T13:28:10.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"update","shortMessageHtmlLink":"update"}},{"before":"230989610635fc40d1237f078600428bd6937727","after":"65b495895ff411e7a208a368796a11624771bfaf","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T13:23:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"ignore cargo audit's own vulnerability","shortMessageHtmlLink":"ignore cargo audit's own vulnerability"}},{"before":"e3fec4f7cbd8a72ad614699999f55d6fbd188785","after":"230989610635fc40d1237f078600428bd6937727","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T12:55:24.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"audit","shortMessageHtmlLink":"audit"}},{"before":"c03582d5c5bb3b7a788580a547a8e243549ec19c","after":"e3fec4f7cbd8a72ad614699999f55d6fbd188785","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T12:18:19.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"style","shortMessageHtmlLink":"style"}},{"before":"9cb07d252839aaca1d264ec54556f86961ad34fa","after":"c03582d5c5bb3b7a788580a547a8e243549ec19c","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T09:14:58.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"mut no longer needed","shortMessageHtmlLink":"mut no longer needed"}},{"before":null,"after":"9cb07d252839aaca1d264ec54556f86961ad34fa","ref":"refs/heads/fix-add-tokens","pushedAt":"2024-04-30T08:53:30.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/48595927?s=80&v=4"},"commit":{"message":"remove enforcement of non special when adding tokens","shortMessageHtmlLink":"remove enforcement of non special when adding tokens"}},{"before":"6682e5c6b82970ca5ed9e27c10ce1fc2c9f1f35e","after":"4992d242a5f4793211a83646ee479b9eb8d6b705","ref":"refs/heads/break_decoder_added_tokens","pushedAt":"2024-04-29T09:39:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/204321?s=80&v=4"},"commit":{"message":"Only prefix space when there's no decoder.","shortMessageHtmlLink":"Only prefix space when there's no decoder."}},{"before":"58bff979275891765d9c5ab0e02e06b9dbf8ed87","after":null,"ref":"refs/heads/dependabot/npm_and_yarn/tokenizers/examples/unstable_wasm/www/webpack-dev-middleware-5.3.4","pushedAt":"2024-04-28T01:50:26.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"dependabot[bot]","name":null,"path":"/apps/dependabot","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/29110?s=80&v=4"}},{"before":"53be059c6063c985d9452e36bb63281368d600ce","after":"6682e5c6b82970ca5ed9e27c10ce1fc2c9f1f35e","ref":"refs/heads/break_decoder_added_tokens","pushedAt":"2024-04-24T16:51:10.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/204321?s=80&v=4"},"commit":{"message":"Clippy.","shortMessageHtmlLink":"Clippy."}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEXCJOwQA","startCursor":null,"endCursor":null}},"title":"Activity · huggingface/tokenizers"}